diff options
62 files changed, 1589 insertions, 250 deletions
diff --git a/Makefile.am b/Makefile.am index abc98e4ed..26f684d59 100644 --- a/Makefile.am +++ b/Makefile.am @@ -39,7 +39,7 @@ dist-hook: sed -i 's/Release:[[:print:]]*/Release: $(RELEASE)/' \ $(distdir)/META -checkstyle: cstyle shellcheck +checkstyle: cstyle shellcheck flake8 cstyle: @find ${top_srcdir} -name '*.[hc]' ! -name 'zfs_config.*' \ @@ -62,6 +62,11 @@ cppcheck: cppcheck --quiet --force --error-exitcode=2 ${top_srcdir}; \ fi +flake8: + @if type flake8 > /dev/null 2>&1; then \ + flake8 ${top_srcdir}; \ + fi + ctags: $(RM) tags find $(top_srcdir) -name .git -prune -o -name '*.[hc]' | xargs ctags diff --git a/cmd/arc_summary/arc_summary.py b/cmd/arc_summary/arc_summary.py index 9f6d8c119..e7448fa5d 100755 --- a/cmd/arc_summary/arc_summary.py +++ b/cmd/arc_summary/arc_summary.py @@ -66,32 +66,17 @@ def get_Kstat(): name, unused, value = kstat.split() Kstat[namespace + name] = D(value) - Kstats = [ - "hw.pagesize", - "hw.physmem", - "kern.maxusers", - "vm.kmem_map_free", - "vm.kmem_map_size", - "vm.kmem_size", - "vm.kmem_size_max", - "vm.kmem_size_min", - "vm.kmem_size_scale", - "vm.stats", - "vm.swap_total", - "vm.swap_reserved", - "kstat.zfs", - "vfs.zfs" - ] Kstat = {} load_proc_kstats('/proc/spl/kstat/zfs/arcstats', - 'kstat.zfs.misc.arcstats.') + 'kstat.zfs.misc.arcstats.') load_proc_kstats('/proc/spl/kstat/zfs/zfetchstats', - 'kstat.zfs.misc.zfetchstats.') + 'kstat.zfs.misc.zfetchstats.') load_proc_kstats('/proc/spl/kstat/zfs/vdev_cache_stats', - 'kstat.zfs.misc.vdev_cache_stats.') + 'kstat.zfs.misc.vdev_cache_stats.') return Kstat + def div1(): sys.stdout.write("\n") for i in range(18): @@ -188,17 +173,17 @@ def get_arc_summary(Kstat): output['memory_throttle_count'] = fHits(memory_throttle_count) - ### ARC Misc. ### + # ARC Misc. deleted = Kstat["kstat.zfs.misc.arcstats.deleted"] mutex_miss = Kstat["kstat.zfs.misc.arcstats.mutex_miss"] - ### ARC Misc. ### + # ARC Misc. output["arc_misc"] = {} output["arc_misc"]["deleted"] = fHits(deleted) output["arc_misc"]['mutex_miss'] = fHits(mutex_miss) output["arc_misc"]['evict_skips'] = fHits(mutex_miss) - ### ARC Sizing ### + # ARC Sizing arc_size = Kstat["kstat.zfs.misc.arcstats.size"] mru_size = Kstat["kstat.zfs.misc.arcstats.p"] target_max_size = Kstat["kstat.zfs.misc.arcstats.c_max"] @@ -207,7 +192,7 @@ def get_arc_summary(Kstat): target_size_ratio = (target_max_size / target_min_size) - ### ARC Sizing ### + # ARC Sizing output['arc_sizing'] = {} output['arc_sizing']['arc_size'] = { 'per': fPerc(arc_size, target_max_size), @@ -226,7 +211,7 @@ def get_arc_summary(Kstat): 'num': fBytes(target_size), } - ### ARC Hash Breakdown ### + # ARC Hash Breakdown output['arc_hash_break'] = {} output['arc_hash_break']['hash_chain_max'] = Kstat[ "kstat.zfs.misc.arcstats.hash_chain_max" @@ -267,7 +252,7 @@ def get_arc_summary(Kstat): 'num': fBytes(mfu_size), } - ### ARC Hash Breakdown ### + # ARC Hash Breakdown hash_chain_max = Kstat["kstat.zfs.misc.arcstats.hash_chain_max"] hash_chains = Kstat["kstat.zfs.misc.arcstats.hash_chains"] hash_collisions = Kstat["kstat.zfs.misc.arcstats.hash_collisions"] @@ -288,25 +273,25 @@ def get_arc_summary(Kstat): def _arc_summary(Kstat): - ### ARC Sizing ### + # ARC Sizing arc = get_arc_summary(Kstat) sys.stdout.write("ARC Summary: (%s)\n" % arc['health']) sys.stdout.write("\tMemory Throttle Count:\t\t\t%s\n" % - arc['memory_throttle_count']) + arc['memory_throttle_count']) sys.stdout.write("\n") - ### ARC Misc. ### + # ARC Misc. sys.stdout.write("ARC Misc:\n") sys.stdout.write("\tDeleted:\t\t\t\t%s\n" % arc['arc_misc']['deleted']) sys.stdout.write("\tMutex Misses:\t\t\t\t%s\n" % - arc['arc_misc']['mutex_miss']) + arc['arc_misc']['mutex_miss']) sys.stdout.write("\tEvict Skips:\t\t\t\t%s\n" % - arc['arc_misc']['mutex_miss']) + arc['arc_misc']['mutex_miss']) sys.stdout.write("\n") - ### ARC Sizing ### + # ARC Sizing sys.stdout.write("ARC Size:\t\t\t\t%s\t%s\n" % ( arc['arc_sizing']['arc_size']['per'], arc['arc_sizing']['arc_size']['num'] @@ -344,21 +329,21 @@ def _arc_summary(Kstat): sys.stdout.write("\n") - ### ARC Hash Breakdown ### + # ARC Hash Breakdown sys.stdout.write("ARC Hash Breakdown:\n") sys.stdout.write("\tElements Max:\t\t\t\t%s\n" % - arc['arc_hash_break']['elements_max']) + arc['arc_hash_break']['elements_max']) sys.stdout.write("\tElements Current:\t\t%s\t%s\n" % ( arc['arc_hash_break']['elements_current']['per'], arc['arc_hash_break']['elements_current']['num'], ) ) sys.stdout.write("\tCollisions:\t\t\t\t%s\n" % - arc['arc_hash_break']['collisions']) + arc['arc_hash_break']['collisions']) sys.stdout.write("\tChain Max:\t\t\t\t%s\n" % - arc['arc_hash_break']['chain_max']) + arc['arc_hash_break']['chain_max']) sys.stdout.write("\tChains:\t\t\t\t\t%s\n" % - arc['arc_hash_break']['chains']) + arc['arc_hash_break']['chains']) def get_arc_efficiency(Kstat): @@ -488,7 +473,7 @@ def _arc_efficiency(Kstat): arc = get_arc_efficiency(Kstat) sys.stdout.write("ARC Total accesses:\t\t\t\t\t%s\n" % - arc['total_accesses']) + arc['total_accesses']) sys.stdout.write("\tCache Hit Ratio:\t\t%s\t%s\n" % ( arc['cache_hit_ratio']['per'], arc['cache_hit_ratio']['num'], @@ -699,7 +684,7 @@ def _l2arc_summary(Kstat): else: sys.stdout.write("(HEALTHY)\n") sys.stdout.write("\tLow Memory Aborts:\t\t\t%s\n" % - arc['low_memory_aborts']) + arc['low_memory_aborts']) sys.stdout.write("\tFree on Write:\t\t\t\t%s\n" % arc['free_on_write']) sys.stdout.write("\tR/W Clashes:\t\t\t\t%s\n" % arc['rw_clashes']) sys.stdout.write("\tBad Checksums:\t\t\t\t%s\n" % arc['bad_checksums']) @@ -707,7 +692,7 @@ def _l2arc_summary(Kstat): sys.stdout.write("\n") sys.stdout.write("L2 ARC Size: (Adaptive)\t\t\t\t%s\n" % - arc["l2_arc_size"]["adative"]) + arc["l2_arc_size"]["adative"]) sys.stdout.write("\tCompressed:\t\t\t%s\t%s\n" % ( arc["l2_arc_size"]["actual"]["per"], arc["l2_arc_size"]["actual"]["num"], @@ -724,13 +709,13 @@ def _l2arc_summary(Kstat): arc["l2_arc_evicts"]["reading"] > 0: sys.stdout.write("L2 ARC Evicts:\n") sys.stdout.write("\tLock Retries:\t\t\t\t%s\n" % - arc["l2_arc_evicts"]['lock_retries']) + arc["l2_arc_evicts"]['lock_retries']) sys.stdout.write("\tUpon Reading:\t\t\t\t%s\n" % - arc["l2_arc_evicts"]["reading"]) + arc["l2_arc_evicts"]["reading"]) sys.stdout.write("\n") sys.stdout.write("L2 ARC Breakdown:\t\t\t\t%s\n" % - arc['l2_arc_breakdown']['value']) + arc['l2_arc_breakdown']['value']) sys.stdout.write("\tHit Ratio:\t\t\t%s\t%s\n" % ( arc['l2_arc_breakdown']['hit_ratio']['per'], arc['l2_arc_breakdown']['hit_ratio']['num'], @@ -744,7 +729,7 @@ def _l2arc_summary(Kstat): ) sys.stdout.write("\tFeeds:\t\t\t\t\t%s\n" % - arc['l2_arc_breakdown']['feeds']) + arc['l2_arc_breakdown']['feeds']) sys.stdout.write("\n") sys.stdout.write("L2 ARC Writes:\n") @@ -803,7 +788,7 @@ def _dmu_summary(Kstat): if arc['zfetch_access_total'] > 0: sys.stdout.write("DMU Prefetch Efficiency:\t\t\t\t\t%s\n" % - arc['dmu']['efficiency']['value']) + arc['dmu']['efficiency']['value']) sys.stdout.write("\tHit Ratio:\t\t\t%s\t%s\n" % ( arc['dmu']['efficiency']['hit_ratio']['per'], arc['dmu']['efficiency']['hit_ratio']['num'], @@ -822,11 +807,11 @@ def get_vdev_summary(Kstat): output = {} vdev_cache_delegations = \ - Kstat["kstat.zfs.misc.vdev_cache_stats.delegations"] + Kstat["kstat.zfs.misc.vdev_cache_stats.delegations"] vdev_cache_misses = Kstat["kstat.zfs.misc.vdev_cache_stats.misses"] vdev_cache_hits = Kstat["kstat.zfs.misc.vdev_cache_stats.hits"] vdev_cache_total = (vdev_cache_misses + vdev_cache_hits + - vdev_cache_delegations) + vdev_cache_delegations) output['vdev_cache_total'] = vdev_cache_total @@ -875,7 +860,8 @@ def _tunable_summary(Kstat): values = {} for name in names: - with open("/sys/module/zfs/parameters/" + name) as f: value = f.read() + with open("/sys/module/zfs/parameters/" + name) as f: + value = f.read() values[name] = value.strip() descriptions = {} @@ -884,7 +870,7 @@ def _tunable_summary(Kstat): try: command = ["/sbin/modinfo", "zfs", "-0"] p = Popen(command, stdin=PIPE, stdout=PIPE, - stderr=PIPE, shell=False, close_fds=True) + stderr=PIPE, shell=False, close_fds=True) p.wait() description_list = p.communicate()[0].strip().split('\0') @@ -899,11 +885,11 @@ def _tunable_summary(Kstat): descriptions[name] = description else: sys.stderr.write("%s: '%s' exited with code %i\n" % - (sys.argv[0], command[0], p.returncode)) + (sys.argv[0], command[0], p.returncode)) sys.stderr.write("Tunable descriptions will be disabled.\n") except OSError as e: sys.stderr.write("%s: Cannot run '%s': %s\n" % - (sys.argv[0], command[0], e.strerror)) + (sys.argv[0], command[0], e.strerror)) sys.stderr.write("Tunable descriptions will be disabled.\n") sys.stdout.write("ZFS Tunable:\n") @@ -942,21 +928,23 @@ def zfs_header(): def usage(): sys.stdout.write("Usage: arc_summary.py [-h] [-a] [-d] [-p PAGE]\n\n") sys.stdout.write("\t -h, --help : " - "Print this help message and exit\n") + "Print this help message and exit\n") sys.stdout.write("\t -a, --alternate : " - "Show an alternate sysctl layout\n") + "Show an alternate sysctl layout\n") sys.stdout.write("\t -d, --description : " - "Show the sysctl descriptions\n") + "Show the sysctl descriptions\n") sys.stdout.write("\t -p PAGE, --page=PAGE : " - "Select a single output page to display,\n") + "Select a single output page to display,\n") sys.stdout.write("\t " - "should be an integer between 1 and " + str(len(unSub)) + "\n\n") + "should be an integer between 1 and " + + str(len(unSub)) + "\n\n") sys.stdout.write("Examples:\n") sys.stdout.write("\tarc_summary.py -a\n") sys.stdout.write("\tarc_summary.py -p 4\n") sys.stdout.write("\tarc_summary.py -ad\n") sys.stdout.write("\tarc_summary.py --page=2\n") + def main(): global show_tunable_descriptions global alternate_tunable_layout @@ -987,9 +975,9 @@ def main(): if 'p' in args: try: pages.append(unSub[int(args['p']) - 1]) - except IndexError as e: + except IndexError: sys.stderr.write('the argument to -p must be between 1 and ' + - str(len(unSub)) + '\n') + str(len(unSub)) + '\n') sys.exit() else: pages = unSub diff --git a/cmd/arcstat/arcstat.py b/cmd/arcstat/arcstat.py index 8bd0d511d..b743fd8bc 100755 --- a/cmd/arcstat/arcstat.py +++ b/cmd/arcstat/arcstat.py @@ -122,7 +122,7 @@ def detailed_usage(): sys.stderr.write("%11s : %s\n" % (key, cols[key][2])) sys.stderr.write("\n") - sys.exit(1) + sys.exit(0) def usage(): @@ -229,15 +229,19 @@ def print_header(): sys.stdout.write("%*s%s" % (cols[col][0], col, sep)) sys.stdout.write("\n") + def get_terminal_lines(): try: - import fcntl, termios, struct + import fcntl + import termios + import struct data = fcntl.ioctl(sys.stdout.fileno(), termios.TIOCGWINSZ, '1234') sz = struct.unpack('hh', data) return sz[0] except: pass + def update_hdr_intr(): global hdr_intr @@ -245,6 +249,7 @@ def update_hdr_intr(): if lines and lines > 3: hdr_intr = lines - 3 + def resize_handler(signum, frame): update_hdr_intr() diff --git a/cmd/dbufstat/dbufstat.py b/cmd/dbufstat/dbufstat.py index 0bda1524e..ceb0160cd 100755 --- a/cmd/dbufstat/dbufstat.py +++ b/cmd/dbufstat/dbufstat.py @@ -143,7 +143,7 @@ def detailed_usage(): sys.stderr.write("%11s : %s\n" % (key, cols[key][2])) sys.stderr.write("\n") - sys.exit(1) + sys.exit(0) def usage(): diff --git a/cmd/zdb/zdb.c b/cmd/zdb/zdb.c index 611e92294..7a0f55f75 100644 --- a/cmd/zdb/zdb.c +++ b/cmd/zdb/zdb.c @@ -1946,11 +1946,13 @@ dump_object(objset_t *os, uint64_t object, int verbosity, int *print_header) } if (verbosity >= 4) { - (void) printf("\tdnode flags: %s%s%s\n", + (void) printf("\tdnode flags: %s%s%s%s\n", (dn->dn_phys->dn_flags & DNODE_FLAG_USED_BYTES) ? "USED_BYTES " : "", (dn->dn_phys->dn_flags & DNODE_FLAG_USERUSED_ACCOUNTED) ? "USERUSED_ACCOUNTED " : "", + (dn->dn_phys->dn_flags & DNODE_FLAG_USEROBJUSED_ACCOUNTED) ? + "USEROBJUSED_ACCOUNTED " : "", (dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR) ? "SPILL_BLKPTR" : ""); (void) printf("\tdnode maxblkid: %llu\n", diff --git a/cmd/zfs/zfs_main.c b/cmd/zfs/zfs_main.c index fedc1a04d..8a49e31e5 100644 --- a/cmd/zfs/zfs_main.c +++ b/cmd/zfs/zfs_main.c @@ -2223,10 +2223,14 @@ enum us_field_types { USFIELD_TYPE, USFIELD_NAME, USFIELD_USED, - USFIELD_QUOTA + USFIELD_QUOTA, + USFIELD_OBJUSED, + USFIELD_OBJQUOTA }; -static char *us_field_hdr[] = { "TYPE", "NAME", "USED", "QUOTA" }; -static char *us_field_names[] = { "type", "name", "used", "quota" }; +static char *us_field_hdr[] = { "TYPE", "NAME", "USED", "QUOTA", + "OBJUSED", "OBJQUOTA" }; +static char *us_field_names[] = { "type", "name", "used", "quota", + "objused", "objquota" }; #define USFIELD_LAST (sizeof (us_field_names) / sizeof (char *)) #define USTYPE_PSX_GRP (1 << 0) @@ -2374,6 +2378,20 @@ compare_nums: return (0); } +static boolean_t +zfs_prop_is_user(unsigned p) +{ + return (p == ZFS_PROP_USERUSED || p == ZFS_PROP_USERQUOTA || + p == ZFS_PROP_USEROBJUSED || p == ZFS_PROP_USEROBJQUOTA); +} + +static boolean_t +zfs_prop_is_group(unsigned p) +{ + return (p == ZFS_PROP_GROUPUSED || p == ZFS_PROP_GROUPQUOTA || + p == ZFS_PROP_GROUPOBJUSED || p == ZFS_PROP_GROUPOBJQUOTA); +} + static inline const char * us_type2str(unsigned field_type) { @@ -2463,7 +2481,7 @@ userspace_cb(void *arg, const char *domain, uid_t rid, uint64_t space) if (cb->cb_sid2posix || domain == NULL || domain[0] == '\0') { /* POSIX or -i */ - if (prop == ZFS_PROP_GROUPUSED || prop == ZFS_PROP_GROUPQUOTA) { + if (zfs_prop_is_group(prop)) { type = USTYPE_PSX_GRP; if (!cb->cb_numname) { struct group *g; @@ -2538,10 +2556,22 @@ userspace_cb(void *arg, const char *domain, uid_t rid, uint64_t space) propname = "used"; if (!nvlist_exists(props, "quota")) (void) nvlist_add_uint64(props, "quota", 0); - } else { + } else if (prop == ZFS_PROP_USERQUOTA || prop == ZFS_PROP_GROUPQUOTA) { propname = "quota"; if (!nvlist_exists(props, "used")) (void) nvlist_add_uint64(props, "used", 0); + } else if (prop == ZFS_PROP_USEROBJUSED || + prop == ZFS_PROP_GROUPOBJUSED) { + propname = "objused"; + if (!nvlist_exists(props, "objquota")) + (void) nvlist_add_uint64(props, "objquota", 0); + } else if (prop == ZFS_PROP_USEROBJQUOTA || + prop == ZFS_PROP_GROUPOBJQUOTA) { + propname = "objquota"; + if (!nvlist_exists(props, "objused")) + (void) nvlist_add_uint64(props, "objused", 0); + } else { + return (-1); } sizeidx = us_field_index(propname); if (sizelen > cb->cb_width[sizeidx]) @@ -2574,7 +2604,7 @@ print_us_node(boolean_t scripted, boolean_t parsable, int *fields, int types, data_type_t type; uint32_t val32; uint64_t val64; - char *strval = NULL; + char *strval = "-"; while ((nvp = nvlist_next_nvpair(nvl, nvp)) != NULL) { if (strcmp(nvpair_name(nvp), @@ -2582,7 +2612,7 @@ print_us_node(boolean_t scripted, boolean_t parsable, int *fields, int types, break; } - type = nvpair_type(nvp); + type = nvp == NULL ? DATA_TYPE_UNKNOWN : nvpair_type(nvp); switch (type) { case DATA_TYPE_UINT32: (void) nvpair_value_uint32(nvp, &val32); @@ -2593,13 +2623,16 @@ print_us_node(boolean_t scripted, boolean_t parsable, int *fields, int types, case DATA_TYPE_STRING: (void) nvpair_value_string(nvp, &strval); break; + case DATA_TYPE_UNKNOWN: + break; default: (void) fprintf(stderr, "invalid data type\n"); } switch (field) { case USFIELD_TYPE: - strval = (char *)us_type2str(val32); + if (type == DATA_TYPE_UINT32) + strval = (char *)us_type2str(val32); break; case USFIELD_NAME: if (type == DATA_TYPE_UINT64) { @@ -2610,6 +2643,8 @@ print_us_node(boolean_t scripted, boolean_t parsable, int *fields, int types, break; case USFIELD_USED: case USFIELD_QUOTA: + case USFIELD_OBJUSED: + case USFIELD_OBJQUOTA: if (type == DATA_TYPE_UINT64) { if (parsable) { (void) sprintf(valstr, "%llu", @@ -2618,7 +2653,8 @@ print_us_node(boolean_t scripted, boolean_t parsable, int *fields, int types, zfs_nicenum(val64, valstr, sizeof (valstr)); } - if (field == USFIELD_QUOTA && + if ((field == USFIELD_QUOTA || + field == USFIELD_OBJQUOTA) && strcmp(valstr, "0") == 0) strval = "none"; else @@ -2690,7 +2726,7 @@ zfs_do_userspace(int argc, char **argv) uu_avl_t *avl_tree; uu_avl_walk_t *walk; char *delim; - char deffields[] = "type,name,used,quota"; + char deffields[] = "type,name,used,quota,objused,objquota"; char *ofield = NULL; char *tfield = NULL; int cfield = 0; @@ -2839,11 +2875,12 @@ zfs_do_userspace(int argc, char **argv) cb.cb_width[i] = strlen(gettext(us_field_hdr[i])); for (p = 0; p < ZFS_NUM_USERQUOTA_PROPS; p++) { - if (((p == ZFS_PROP_USERUSED || p == ZFS_PROP_USERQUOTA) && + if ((zfs_prop_is_user(p) && !(types & (USTYPE_PSX_USR | USTYPE_SMB_USR))) || - ((p == ZFS_PROP_GROUPUSED || p == ZFS_PROP_GROUPQUOTA) && + (zfs_prop_is_group(p) && !(types & (USTYPE_PSX_GRP | USTYPE_SMB_GRP)))) continue; + cb.cb_prop = p; if ((ret = zfs_userspace(zhp, p, userspace_cb, &cb)) != 0) return (ret); @@ -4099,6 +4136,11 @@ zfs_do_receive(int argc, char **argv) #define ZFS_DELEG_PERM_GROUPQUOTA "groupquota" #define ZFS_DELEG_PERM_USERUSED "userused" #define ZFS_DELEG_PERM_GROUPUSED "groupused" +#define ZFS_DELEG_PERM_USEROBJQUOTA "userobjquota" +#define ZFS_DELEG_PERM_GROUPOBJQUOTA "groupobjquota" +#define ZFS_DELEG_PERM_USEROBJUSED "userobjused" +#define ZFS_DELEG_PERM_GROUPOBJUSED "groupobjused" + #define ZFS_DELEG_PERM_HOLD "hold" #define ZFS_DELEG_PERM_RELEASE "release" #define ZFS_DELEG_PERM_DIFF "diff" @@ -4129,6 +4171,10 @@ static zfs_deleg_perm_tab_t zfs_deleg_perm_tbl[] = { { ZFS_DELEG_PERM_USERPROP, ZFS_DELEG_NOTE_USERPROP }, { ZFS_DELEG_PERM_USERQUOTA, ZFS_DELEG_NOTE_USERQUOTA }, { ZFS_DELEG_PERM_USERUSED, ZFS_DELEG_NOTE_USERUSED }, + { ZFS_DELEG_PERM_USEROBJQUOTA, ZFS_DELEG_NOTE_USEROBJQUOTA }, + { ZFS_DELEG_PERM_USEROBJUSED, ZFS_DELEG_NOTE_USEROBJUSED }, + { ZFS_DELEG_PERM_GROUPOBJQUOTA, ZFS_DELEG_NOTE_GROUPOBJQUOTA }, + { ZFS_DELEG_PERM_GROUPOBJUSED, ZFS_DELEG_NOTE_GROUPOBJUSED }, { NULL, ZFS_DELEG_NOTE_NONE } }; @@ -4206,6 +4252,10 @@ deleg_perm_type(zfs_deleg_note_t note) case ZFS_DELEG_NOTE_USERPROP: case ZFS_DELEG_NOTE_USERQUOTA: case ZFS_DELEG_NOTE_USERUSED: + case ZFS_DELEG_NOTE_USEROBJQUOTA: + case ZFS_DELEG_NOTE_USEROBJUSED: + case ZFS_DELEG_NOTE_GROUPOBJQUOTA: + case ZFS_DELEG_NOTE_GROUPOBJUSED: /* other */ return (gettext("other")); default: @@ -4709,6 +4759,19 @@ deleg_perm_comment(zfs_deleg_note_t note) case ZFS_DELEG_NOTE_USERUSED: str = gettext("Allows reading any userused@... property"); break; + case ZFS_DELEG_NOTE_USEROBJQUOTA: + str = gettext("Allows accessing any userobjquota@... property"); + break; + case ZFS_DELEG_NOTE_GROUPOBJQUOTA: + str = gettext("Allows accessing any \n\t\t\t\t" + "groupobjquota@... property"); + break; + case ZFS_DELEG_NOTE_GROUPOBJUSED: + str = gettext("Allows reading any groupobjused@... property"); + break; + case ZFS_DELEG_NOTE_USEROBJUSED: + str = gettext("Allows reading any userobjused@... property"); + break; /* other */ default: str = ""; diff --git a/configure.ac b/configure.ac index edcf29958..f01a4d872 100644 --- a/configure.ac +++ b/configure.ac @@ -274,6 +274,7 @@ AC_CONFIG_FILES([ tests/zfs-tests/tests/functional/threadsappend/Makefile tests/zfs-tests/tests/functional/truncate/Makefile tests/zfs-tests/tests/functional/userquota/Makefile + tests/zfs-tests/tests/functional/upgrade/Makefile tests/zfs-tests/tests/functional/vdev_zaps/Makefile tests/zfs-tests/tests/functional/write_dirs/Makefile tests/zfs-tests/tests/functional/xattr/Makefile diff --git a/contrib/dracut/90zfs/mount-zfs.sh.in b/contrib/dracut/90zfs/mount-zfs.sh.in index 51e107bb3..3b6be314c 100755 --- a/contrib/dracut/90zfs/mount-zfs.sh.in +++ b/contrib/dracut/90zfs/mount-zfs.sh.in @@ -10,17 +10,29 @@ case "${root}" in *) return ;; esac -if command -v systemctl >/dev/null; then - # If sysroot.mount exists, the initial RAM disk configured - # it to mount ZFS on root. In that case, we bail early. - loadstate="$(systemctl --system --show -p LoadState sysroot.mount || true)" - if [ "${loadstate}" = "LoadState=not-found" -o "${loadstate}" = "" ] ; then - info "ZFS: sysroot.mount absent, mounting root with mount-zfs.sh" - else - info "ZFS: sysroot.mount present, delegating root mount to it" - return - fi +GENERATOR_FILE=/run/systemd/generator/sysroot.mount +GENERATOR_EXTENSION=/run/systemd/generator/sysroot.mount.d/zfs-enhancement.conf + +if [ -e "$GENERATOR_FILE" -a -e "$GENERATOR_EXTENSION" ] ; then + # If the ZFS sysroot.mount flag exists, the initial RAM disk configured + # it to mount ZFS on root. In that case, we bail early. This flag + # file gets created by the zfs-generator program upon successful run. + info "ZFS: There is a sysroot.mount and zfs-generator has extended it." + info "ZFS: Delegating root mount to sysroot.mount." + # Let us tell the initrd to run on shutdown. + # We have a shutdown hook to run + # because we imported the pool. + need_shutdown + # We now prevent Dracut from running this thing again. + for zfsmounthook in "$hookdir"/mount/*zfs* ; do + if [ -f "$zfsmounthook" ] ; then + rm -f "$zfsmounthook" + fi + done + return fi +info "ZFS: No sysroot.mount exists or zfs-generator did not extend it." +info "ZFS: Mounting root with the traditional mount-zfs.sh instead." # Delay until all required block devices are present. udevadm settle @@ -45,6 +57,10 @@ ZFS_DATASET="${ZFS_DATASET:-${root#zfs:}}" ZFS_POOL="${ZFS_DATASET%%/*}" if import_pool "${ZFS_POOL}" ; then + # Let us tell the initrd to run on shutdown. + # We have a shutdown hook to run + # because we imported the pool. + need_shutdown info "ZFS: Mounting dataset ${ZFS_DATASET}..." if mount_dataset "${ZFS_DATASET}" ; then ROOTFS_MOUNTED=yes @@ -53,4 +69,3 @@ if import_pool "${ZFS_POOL}" ; then fi rootok=0 -need_shutdown diff --git a/contrib/dracut/90zfs/zfs-generator.sh.in b/contrib/dracut/90zfs/zfs-generator.sh.in index aa12fb856..c6384f583 100755 --- a/contrib/dracut/90zfs/zfs-generator.sh.in +++ b/contrib/dracut/90zfs/zfs-generator.sh.in @@ -1,21 +1,32 @@ #!/bin/bash +echo "zfs-generator: starting" >> /dev/kmsg + GENERATOR_DIR="$1" -[ -z "$GENERATOR_DIR" ] && exit 1 +[ -n "$GENERATOR_DIR" ] || { + echo "zfs-generator: no generator directory specified, exiting" >> /dev/kmsg + exit 1 +} [ -f /lib/dracut-lib.sh ] && dracutlib=/lib/dracut-lib.sh [ -f /usr/lib/dracut/modules.d/99base/dracut-lib.sh ] && dracutlib=/usr/lib/dracut/modules.d/99base/dracut-lib.sh -type getarg >/dev/null 2>&1 || . "$dracutlib" +type getarg >/dev/null 2>&1 || { + echo "zfs-generator: loading Dracut library from $dracutlib" >> /dev/kmsg + . "$dracutlib" +} [ -z "$root" ] && root=$(getarg root=) [ -z "$rootfstype" ] && rootfstype=$(getarg rootfstype=) [ -z "$rootflags" ] && rootflags=$(getarg rootflags=) +# If root is not ZFS= or zfs: or rootfstype is not zfs +# then we are not supposed to handle it. [ "${root##zfs:}" = "${root}" -a "${root##ZFS=}" = "${root}" -a "$rootfstype" != "zfs" ] && exit 0 -# If root is set to zfs:AUTO, then we know sysroot.mount will not be generated -# so we have no need to enhance it. -# See https://github.com/zfsonlinux/zfs/pull/4558#discussion_r61118952 for details. +# If root is set to zfs:AUTO, then we are also not +# supposed to handle it, and it should be handled +# by the traditional Dracut mount hook. +# See https://github.com/zfsonlinux/zfs/pull/4558#discussion_r61118952 if [ "${root}" = "zfs:AUTO" ] ; then exit 0 fi @@ -32,16 +43,23 @@ fi root="${root##zfs:}" root="${root##ZFS=}" +echo "zfs-generator: writing extension for sysroot.mount to $GENERATOR_DIR"/sysroot.mount.d/zfs-enhancement.conf >> /dev/kmsg + [ -d "$GENERATOR_DIR" ] || mkdir "$GENERATOR_DIR" -[ -d "$GENERATOR_DIR/sysroot.mount.d" ] || mkdir "$GENERATOR_DIR/sysroot.mount.d" +[ -d "$GENERATOR_DIR"/sysroot.mount.d ] || mkdir "$GENERATOR_DIR"/sysroot.mount.d { echo "[Unit]" + echo "Before=initrd-root-fs.target" echo "After=zfs-import-scan.service" echo "After=zfs-import-cache.service" - echo "" echo "[Mount]" echo "What=${root}" echo "Type=${rootfstype}" echo "Options=${rootflags}" -} > "$GENERATOR_DIR/sysroot.mount.d/zfs-enhancement.conf" +} > "$GENERATOR_DIR"/sysroot.mount.d/zfs-enhancement.conf + +[ -d "$GENERATOR_DIR"/initrd-root-fs.target.requires ] || mkdir -p "$GENERATOR_DIR"/initrd-root-fs.target.requires +ln -s ../sysroot.mount "$GENERATOR_DIR"/initrd-root-fs.target.requires/sysroot.mount + +echo "zfs-generator: finished" >> /dev/kmsg diff --git a/include/sys/dmu.h b/include/sys/dmu.h index b67acb52c..ec89e9018 100644 --- a/include/sys/dmu.h +++ b/include/sys/dmu.h @@ -257,6 +257,12 @@ void zfs_znode_byteswap(void *buf, size_t size); #define DMU_GROUPUSED_OBJECT (-2ULL) /* + * Zap prefix for object accounting in DMU_{USER,GROUP}USED_OBJECT. + */ +#define DMU_OBJACCT_PREFIX "obj-" +#define DMU_OBJACCT_PREFIX_LEN 4 + +/* * artificial blkids for bonus buffer and spill blocks */ #define DMU_BONUS_BLKID (-1ULL) diff --git a/include/sys/dmu_objset.h b/include/sys/dmu_objset.h index 1674897c2..68fb5cffb 100644 --- a/include/sys/dmu_objset.h +++ b/include/sys/dmu_objset.h @@ -56,6 +56,7 @@ struct dmu_tx; (arc_buf_size(buf) > OBJSET_OLD_PHYS_SIZE) #define OBJSET_FLAG_USERACCOUNTING_COMPLETE (1ULL<<0) +#define OBJSET_FLAG_USEROBJACCOUNTING_COMPLETE (1ULL<<1) typedef struct objset_phys { dnode_phys_t os_meta_dnode; @@ -68,6 +69,8 @@ typedef struct objset_phys { dnode_phys_t os_groupused_dnode; } objset_phys_t; +typedef int (*dmu_objset_upgrade_cb_t)(objset_t *); + struct objset { /* Immutable: */ struct dsl_dataset *os_dsl_dataset; @@ -125,6 +128,13 @@ struct objset { kmutex_t os_user_ptr_lock; void *os_user_ptr; sa_os_t *os_sa; + + /* kernel thread to upgrade this dataset */ + kmutex_t os_upgrade_lock; + taskqid_t os_upgrade_id; + dmu_objset_upgrade_cb_t os_upgrade_cb; + boolean_t os_upgrade_exit; + int os_upgrade_status; }; #define DMU_META_OBJSET 0 @@ -173,6 +183,17 @@ void dmu_objset_userquota_get_ids(dnode_t *dn, boolean_t before, dmu_tx_t *tx); boolean_t dmu_objset_userused_enabled(objset_t *os); int dmu_objset_userspace_upgrade(objset_t *os); boolean_t dmu_objset_userspace_present(objset_t *os); +boolean_t dmu_objset_userobjused_enabled(objset_t *os); +void dmu_objset_userobjspace_upgrade(objset_t *os); +boolean_t dmu_objset_userobjspace_present(objset_t *os); + +static inline boolean_t dmu_objset_userobjspace_upgradable(objset_t *os) +{ + return (dmu_objset_type(os) == DMU_OST_ZFS && + dmu_objset_userobjused_enabled(os) && + !dmu_objset_userobjspace_present(os)); +} + int dmu_fsname(const char *snapname, char *buf); void dmu_objset_evict_done(objset_t *os); diff --git a/include/sys/dnode.h b/include/sys/dnode.h index fe36e5989..fe4cd3e4b 100644 --- a/include/sys/dnode.h +++ b/include/sys/dnode.h @@ -126,11 +126,14 @@ enum dnode_dirtycontext { }; /* Is dn_used in bytes? if not, it's in multiples of SPA_MINBLOCKSIZE */ -#define DNODE_FLAG_USED_BYTES (1<<0) -#define DNODE_FLAG_USERUSED_ACCOUNTED (1<<1) +#define DNODE_FLAG_USED_BYTES (1 << 0) +#define DNODE_FLAG_USERUSED_ACCOUNTED (1 << 1) /* Does dnode have a SA spill blkptr in bonus? */ -#define DNODE_FLAG_SPILL_BLKPTR (1<<2) +#define DNODE_FLAG_SPILL_BLKPTR (1 << 2) + +/* User/Group dnode accounting */ +#define DNODE_FLAG_USEROBJUSED_ACCOUNTED (1 << 3) typedef struct dnode_phys { uint8_t dn_type; /* dmu_object_type_t */ diff --git a/include/sys/dsl_deleg.h b/include/sys/dsl_deleg.h index 59e8e0555..d399d1da9 100644 --- a/include/sys/dsl_deleg.h +++ b/include/sys/dsl_deleg.h @@ -51,8 +51,12 @@ extern "C" { #define ZFS_DELEG_PERM_VSCAN "vscan" #define ZFS_DELEG_PERM_USERQUOTA "userquota" #define ZFS_DELEG_PERM_GROUPQUOTA "groupquota" +#define ZFS_DELEG_PERM_USEROBJQUOTA "userobjquota" +#define ZFS_DELEG_PERM_GROUPOBJQUOTA "groupobjquota" #define ZFS_DELEG_PERM_USERUSED "userused" #define ZFS_DELEG_PERM_GROUPUSED "groupused" +#define ZFS_DELEG_PERM_USEROBJUSED "userobjused" +#define ZFS_DELEG_PERM_GROUPOBJUSED "groupobjused" #define ZFS_DELEG_PERM_HOLD "hold" #define ZFS_DELEG_PERM_RELEASE "release" #define ZFS_DELEG_PERM_DIFF "diff" diff --git a/include/sys/fs/zfs.h b/include/sys/fs/zfs.h index 9d3008934..5c93f53de 100644 --- a/include/sys/fs/zfs.h +++ b/include/sys/fs/zfs.h @@ -171,6 +171,10 @@ typedef enum { ZFS_PROP_USERQUOTA, ZFS_PROP_GROUPUSED, ZFS_PROP_GROUPQUOTA, + ZFS_PROP_USEROBJUSED, + ZFS_PROP_USEROBJQUOTA, + ZFS_PROP_GROUPOBJUSED, + ZFS_PROP_GROUPOBJQUOTA, ZFS_NUM_USERQUOTA_PROPS } zfs_userquota_prop_t; diff --git a/include/sys/spa_impl.h b/include/sys/spa_impl.h index 7b9e1ee0c..cb1d16ad5 100644 --- a/include/sys/spa_impl.h +++ b/include/sys/spa_impl.h @@ -277,6 +277,8 @@ struct spa { */ spa_config_lock_t spa_config_lock[SCL_LOCKS]; /* config changes */ refcount_t spa_refcount; /* number of opens */ + + taskq_t *spa_upgrade_taskq; /* taskq for upgrade jobs */ }; extern char *spa_config_path; diff --git a/include/sys/zfs_vfsops.h b/include/sys/zfs_vfsops.h index efaefdacc..b59ace5b1 100644 --- a/include/sys/zfs_vfsops.h +++ b/include/sys/zfs_vfsops.h @@ -110,6 +110,8 @@ typedef struct zfs_sb { kmutex_t z_lock; uint64_t z_userquota_obj; uint64_t z_groupquota_obj; + uint64_t z_userobjquota_obj; + uint64_t z_groupobjquota_obj; uint64_t z_replay_eof; /* New end of file - replay only */ sa_attr_type_t *z_attr_table; /* SA attr mapping->id */ uint64_t z_hold_size; /* znode hold array size */ @@ -190,6 +192,8 @@ extern boolean_t zfs_owner_overquota(zfs_sb_t *zsb, struct znode *, boolean_t isgroup); extern boolean_t zfs_fuid_overquota(zfs_sb_t *zsb, boolean_t isgroup, uint64_t fuid); +extern boolean_t zfs_fuid_overobjquota(zfs_sb_t *zsb, boolean_t isgroup, + uint64_t fuid); extern int zfs_set_version(zfs_sb_t *zsb, uint64_t newvers); extern int zfs_get_zplprop(objset_t *os, zfs_prop_t prop, uint64_t *value); diff --git a/include/zfeature_common.h b/include/zfeature_common.h index f05480181..acf76381b 100644 --- a/include/zfeature_common.h +++ b/include/zfeature_common.h @@ -54,6 +54,7 @@ typedef enum spa_feature { SPA_FEATURE_SHA512, SPA_FEATURE_SKEIN, SPA_FEATURE_EDONR, + SPA_FEATURE_USEROBJ_ACCOUNTING, SPA_FEATURES } spa_feature_t; diff --git a/include/zfs_deleg.h b/include/zfs_deleg.h index 16133c59f..95db9921f 100644 --- a/include/zfs_deleg.h +++ b/include/zfs_deleg.h @@ -63,6 +63,10 @@ typedef enum { ZFS_DELEG_NOTE_GROUPQUOTA, ZFS_DELEG_NOTE_USERUSED, ZFS_DELEG_NOTE_GROUPUSED, + ZFS_DELEG_NOTE_USEROBJQUOTA, + ZFS_DELEG_NOTE_GROUPOBJQUOTA, + ZFS_DELEG_NOTE_USEROBJUSED, + ZFS_DELEG_NOTE_GROUPOBJUSED, ZFS_DELEG_NOTE_HOLD, ZFS_DELEG_NOTE_RELEASE, ZFS_DELEG_NOTE_DIFF, diff --git a/lib/libzfs/libzfs_dataset.c b/lib/libzfs/libzfs_dataset.c index 5ecf96985..7d8179d1f 100755 --- a/lib/libzfs/libzfs_dataset.c +++ b/lib/libzfs/libzfs_dataset.c @@ -944,7 +944,9 @@ zfs_valid_proplist(libzfs_handle_t *hdl, zfs_type_t type, nvlist_t *nvl, } if (uqtype != ZFS_PROP_USERQUOTA && - uqtype != ZFS_PROP_GROUPQUOTA) { + uqtype != ZFS_PROP_GROUPQUOTA && + uqtype != ZFS_PROP_USEROBJQUOTA && + uqtype != ZFS_PROP_GROUPOBJQUOTA) { zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "'%s' is readonly"), propname); @@ -2741,8 +2743,12 @@ userquota_propname_decode(const char *propname, boolean_t zoned, return (EINVAL); *typep = type; - isuser = (type == ZFS_PROP_USERQUOTA || type == ZFS_PROP_USERUSED); - isgroup = (type == ZFS_PROP_GROUPQUOTA || type == ZFS_PROP_GROUPUSED); + isuser = (type == ZFS_PROP_USERQUOTA || type == ZFS_PROP_USERUSED || + type == ZFS_PROP_USEROBJQUOTA || + type == ZFS_PROP_USEROBJUSED); + isgroup = (type == ZFS_PROP_GROUPQUOTA || type == ZFS_PROP_GROUPUSED || + type == ZFS_PROP_GROUPOBJQUOTA || + type == ZFS_PROP_GROUPOBJUSED); cp = strchr(propname, '@') + 1; @@ -2875,7 +2881,8 @@ zfs_prop_get_userquota(zfs_handle_t *zhp, const char *propname, (void) snprintf(propbuf, proplen, "%llu", (u_longlong_t)propvalue); } else if (propvalue == 0 && - (type == ZFS_PROP_USERQUOTA || type == ZFS_PROP_GROUPQUOTA)) { + (type == ZFS_PROP_USERQUOTA || type == ZFS_PROP_GROUPQUOTA || + type == ZFS_PROP_USEROBJQUOTA || type == ZFS_PROP_GROUPOBJQUOTA)) { (void) strlcpy(propbuf, "none", proplen); } else { zfs_nicenum(propvalue, propbuf, proplen); @@ -4333,6 +4340,13 @@ zfs_userspace(zfs_handle_t *zhp, zfs_userquota_prop_t type, if (zfs_ioctl(hdl, ZFS_IOC_USERSPACE_MANY, &zc) != 0) { char errbuf[1024]; + if ((errno == ENOTSUP && + (type == ZFS_PROP_USEROBJUSED || + type == ZFS_PROP_GROUPOBJUSED || + type == ZFS_PROP_USEROBJQUOTA || + type == ZFS_PROP_GROUPOBJQUOTA))) + break; + (void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN, "cannot get used/quota for %s"), zc.zc_name); diff --git a/lib/libzfs/libzfs_sendrecv.c b/lib/libzfs/libzfs_sendrecv.c index f70e34107..ff84a1acd 100644 --- a/lib/libzfs/libzfs_sendrecv.c +++ b/lib/libzfs/libzfs_sendrecv.c @@ -3110,6 +3110,7 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap, ENOENT); if (stream_avl != NULL) { + nvlist_t *lookup = NULL; nvlist_t *fs = fsavl_find(stream_avl, drrb->drr_toguid, &snapname); @@ -3125,6 +3126,10 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap, VERIFY(0 == nvlist_add_uint64(props, zfs_prop_to_name(ZFS_PROP_CANMOUNT), 0)); } + if (0 == nvlist_lookup_nvlist(fs, "snapprops", &lookup)) { + VERIFY(0 == nvlist_lookup_nvlist(lookup, + snapname, &snapprops_nvlist)); + } } cp = NULL; diff --git a/man/man5/zpool-features.5 b/man/man5/zpool-features.5 index ffefd4129..ccc7ab47e 100644 --- a/man/man5/zpool-features.5 +++ b/man/man5/zpool-features.5 @@ -563,5 +563,31 @@ Booting off of pools using \fBedonr\fR is \fBNOT\fR supported -- any attempt to enable \fBedonr\fR on a root pool will fail with an error. +.sp +.ne 2 +.na +\fB\fBuserobj_accounting\fR\fR +.ad +.RS 4n +.TS +l l . +GUID org.zfsonlinux:userobj_accounting +READ\-ONLY COMPATIBLE yes +DEPENDENCIES extensible_dataset +.TE + +This feature allows administrators to account the object usage information +by user and group. + +This feature becomes \fBactive\fR as soon as it is enabled and will never +return to being \fBenabled\fR. Each filesystem will be upgraded automatically +when remounted, or when new files are created under that filesystem. +The upgrade can also be started manually on filesystems by running +`zfs set version=current <pool/fs>`. The upgrade process runs in the background +and may take a while to complete for filesystems containing a large number of +files. + +.RE + .SH "SEE ALSO" \fBzpool\fR(8) diff --git a/man/man8/zfs.8 b/man/man8/zfs.8 index e543ba51d..d8c150629 100644 --- a/man/man8/zfs.8 +++ b/man/man8/zfs.8 @@ -663,6 +663,8 @@ The amount of space consumed by snapshots of this dataset. In particular, it is .sp .ne 2 .na +\fB\fBuserobjused@\fR\fIuser\fR\fR +.br \fB\fBuserused@\fR\fIuser\fR\fR .ad .sp .6 @@ -699,6 +701,11 @@ The \fBuserused@\fR... properties are not displayed by \fBzfs get all\fR. The us .RE Files created on Linux always have POSIX owners. +.RS 4n +The \fBuserobjused\fR is similar to \fBuserused\fR but instead it counts the number of objects consumed by \fIuser\fR. This feature doesn't count the internal objects used by ZFS, therefore it may under count a few objects comparing with the results of third-party tool such as \fBdfs -i\fR. +When the property \fBxattr=on\fR is set on a fileset, ZFS will create additional objects per-file to store extended attributes. These additional objects are reflected in the \fBuserobjused\fR value and are counted against the user's \fBuserobjquota\fR. When a filesystem is configured to use \fBxattr=sa\fR no additional internal objects are required. +.RE + .sp .ne 2 .na @@ -713,6 +720,8 @@ This property is set to the number of user holds on this snapshot. User holds ar .ne 2 .na \fB\fBgroupused@\fR\fIgroup\fR\fR +.br +\fB\fBgroupobjused@\fR\fIgroup\fR\fR .ad .sp .6 .RS 4n @@ -721,6 +730,11 @@ The amount of space consumed by the specified group in this dataset. Space is ch Unprivileged users can only access their own groups' space usage. The root user, or a user who has been granted the \fBgroupused\fR privilege with \fBzfs allow\fR, can access all groups' usage. .RE +.RS 4n +The \fBgroupobjused\fR is similar to \fBgroupused\fR but instead it counts the number of objects consumed by \fIgroup\fR. +When the property \fBxattr=on\fR is set on a fileset, ZFS will create additional objects per-file to store extended attributes. These additional objects are reflected in the \fBgroupobjused\fR value and are counted against the group's \fBgroupobjquota.\fR. When a filesystem is configured to use \fBxattr=sa\fR no additional internal objects are required. +.RE + .sp .ne 2 .na @@ -1081,6 +1095,8 @@ a zone. This feature must be enabled to be used (see \fBzpool-features\fR(5)). .ne 2 .na \fB\fBuserquota@\fR\fIuser\fR=\fBnone\fR | \fIsize\fR\fR +.br +\fB\fBuserobjquota@\fR\fIuser\fR=\fBnone\fR | \fIcount\fR\fR .ad .sp .6 .RS 4n @@ -1118,16 +1134,26 @@ This property is not available on volumes, on file systems before version 4, or .RE Files created on Linux always have POSIX owners. +.RS 4 +The \fBuserobjquota\fR is similar to \fBuserquota\fR but it limits the number of objects a \fIuser\fR can create. +Please refer to \fBuserobjused\fR for more information about how ZFS counts object usage. +.RE + .sp .ne 2 .na \fB\fBgroupquota@\fR\fIgroup\fR=\fBnone\fR\fR | \fIsize\fR +.br +\fB\fBgroupobjquota@\fR\fIgroup\fR=\fBnone\fR\fR | \fIcount\fR .ad .sp .6 .RS 4n Limits the amount of space consumed by the specified group. Group space consumption is identified by the \fBuserquota@\fR\fIuser\fR property. .sp Unprivileged users can access only their own groups' space usage. The root user, or a user who has been granted the \fBgroupquota\fR privilege with \fBzfs allow\fR, can get and set all groups' quotas. + +The \fBgroupobjquota\fR is similar to \fBgroupquota\fR but it limits that the \fIgroup\fR can consume \fIcount\fR number of objects at most. +Please refer to \fBuserobjused\fR for more information about how zfs counts object usage. .RE .sp @@ -2386,8 +2412,8 @@ Upgrades to the specified \fIversion\fR. If the \fB-V\fR flag is not specified, .sp .6 .RS 4n Displays space consumed by, and quotas on, each user in the specified -filesystem or snapshot. This corresponds to the \fBuserused@\fR\fIuser\fR and -\fBuserquota@\fR\fIuser\fR properties. +filesystem or snapshot. This corresponds to the \fBuserused@\fR\fIuser\fR, \fBuserobjused@\fR\fIuser\fR, +\fBuserquota@\fR\fIuser\fR, and \fBuserobjquota@\fR\fIuser\fR properties. .sp .ne 2 .na @@ -3141,10 +3167,14 @@ send subcommand share subcommand Allows sharing file systems over NFS or SMB protocols snapshot subcommand Must also have the 'mount' ability +groupobjquota other Allows accessing any groupobjquota@... property groupquota other Allows accessing any groupquota@... property +groupobjused other Allows reading any groupobjused@... property groupused other Allows reading any groupused@... property userprop other Allows changing any user property +userobjquota other Allows accessing any userobjquota@... property userquota other Allows accessing any userquota@... property +userobjused other Allows reading any userobjused@... property userused other Allows reading any userused@... property acltype property diff --git a/module/zcommon/zfs_deleg.c b/module/zcommon/zfs_deleg.c index f6e41da9d..647a24e5f 100644 --- a/module/zcommon/zfs_deleg.c +++ b/module/zcommon/zfs_deleg.c @@ -62,6 +62,10 @@ zfs_deleg_perm_tab_t zfs_deleg_perm_tab[] = { {ZFS_DELEG_PERM_GROUPQUOTA}, {ZFS_DELEG_PERM_USERUSED}, {ZFS_DELEG_PERM_GROUPUSED}, + {ZFS_DELEG_PERM_USEROBJQUOTA}, + {ZFS_DELEG_PERM_GROUPOBJQUOTA}, + {ZFS_DELEG_PERM_USEROBJUSED}, + {ZFS_DELEG_PERM_GROUPOBJUSED}, {ZFS_DELEG_PERM_HOLD}, {ZFS_DELEG_PERM_RELEASE}, {NULL} diff --git a/module/zcommon/zfs_prop.c b/module/zcommon/zfs_prop.c index 029075ebe..1802750f9 100644 --- a/module/zcommon/zfs_prop.c +++ b/module/zcommon/zfs_prop.c @@ -52,7 +52,11 @@ const char *zfs_userquota_prop_prefixes[] = { "userused@", "userquota@", "groupused@", - "groupquota@" + "groupquota@", + "userobjused@", + "userobjquota@", + "groupobjused@", + "groupobjquota@" }; zprop_desc_t * diff --git a/module/zfs/dmu_objset.c b/module/zfs/dmu_objset.c index 970ee4f08..b1f05cb61 100644 --- a/module/zfs/dmu_objset.c +++ b/module/zfs/dmu_objset.c @@ -31,6 +31,7 @@ /* Portions Copyright 2010 Robert Milkowski */ +#include <sys/zfeature.h> #include <sys/cred.h> #include <sys/zfs_context.h> #include <sys/dmu_objset.h> @@ -53,6 +54,7 @@ #include <sys/dsl_destroy.h> #include <sys/vdev.h> #include <sys/policy.h> +#include <sys/spa_impl.h> /* * Needed to close a window in dnode_move() that allows the objset to be freed @@ -77,6 +79,9 @@ int dmu_rescan_dnode_threshold = 1 << DN_MAX_INDBLKSHIFT; static void dmu_objset_find_dp_cb(void *arg); +static void dmu_objset_upgrade(objset_t *os, dmu_objset_upgrade_cb_t cb); +static void dmu_objset_upgrade_stop(objset_t *os); + void dmu_objset_init(void) { @@ -519,6 +524,8 @@ dmu_objset_open_impl(spa_t *spa, dsl_dataset_t *ds, blkptr_t *bp, DMU_GROUPUSED_OBJECT, &os->os_groupused_dnode); } + mutex_init(&os->os_upgrade_lock, NULL, MUTEX_DEFAULT, NULL); + *osp = os; return (0); } @@ -625,6 +632,9 @@ dmu_objset_own(const char *name, dmu_objset_type_t type, err = dmu_objset_own_impl(ds, type, readonly, tag, osp); dsl_pool_rele(dp, FTAG); + if (err == 0 && dmu_objset_userobjspace_upgradable(*osp)) + dmu_objset_userobjspace_upgrade(*osp); + return (err); } @@ -685,6 +695,10 @@ dmu_objset_refresh_ownership(objset_t *os, void *tag) void dmu_objset_disown(objset_t *os, void *tag) { + /* + * Stop upgrading thread + */ + dmu_objset_upgrade_stop(os); dsl_dataset_disown(os->os_dsl_dataset, tag); } @@ -859,6 +873,12 @@ dmu_objset_create_impl(spa_t *spa, dsl_dataset_t *ds, blkptr_t *bp, os->os_phys->os_type = type; if (dmu_objset_userused_enabled(os)) { os->os_phys->os_flags |= OBJSET_FLAG_USERACCOUNTING_COMPLETE; + if (dmu_objset_userobjused_enabled(os)) { + ds->ds_feature_activation_needed[ + SPA_FEATURE_USEROBJ_ACCOUNTING] = B_TRUE; + os->os_phys->os_flags |= + OBJSET_FLAG_USEROBJACCOUNTING_COMPLETE; + } os->os_flags = os->os_phys->os_flags; } @@ -1068,6 +1088,60 @@ dmu_objset_snapshot_one(const char *fsname, const char *snapname) } static void +dmu_objset_upgrade_task_cb(void *data) +{ + objset_t *os = data; + + mutex_enter(&os->os_upgrade_lock); + os->os_upgrade_status = EINTR; + if (!os->os_upgrade_exit) { + mutex_exit(&os->os_upgrade_lock); + + os->os_upgrade_status = os->os_upgrade_cb(os); + mutex_enter(&os->os_upgrade_lock); + } + os->os_upgrade_exit = B_TRUE; + os->os_upgrade_id = 0; + mutex_exit(&os->os_upgrade_lock); +} + +static void +dmu_objset_upgrade(objset_t *os, dmu_objset_upgrade_cb_t cb) +{ + if (os->os_upgrade_id != 0) + return; + + mutex_enter(&os->os_upgrade_lock); + if (os->os_upgrade_id == 0 && os->os_upgrade_status == 0) { + os->os_upgrade_exit = B_FALSE; + os->os_upgrade_cb = cb; + os->os_upgrade_id = taskq_dispatch( + os->os_spa->spa_upgrade_taskq, + dmu_objset_upgrade_task_cb, os, TQ_SLEEP); + if (os->os_upgrade_id == 0) + os->os_upgrade_status = ENOMEM; + } + mutex_exit(&os->os_upgrade_lock); +} + +static void +dmu_objset_upgrade_stop(objset_t *os) +{ + mutex_enter(&os->os_upgrade_lock); + os->os_upgrade_exit = B_TRUE; + if (os->os_upgrade_id != 0) { + taskqid_t id = os->os_upgrade_id; + + os->os_upgrade_id = 0; + mutex_exit(&os->os_upgrade_lock); + + taskq_cancel_id(os->os_spa->spa_upgrade_taskq, id); + } else { + mutex_exit(&os->os_upgrade_lock); + } +} + +static void dmu_objset_sync_dnodes(list_t *list, list_t *newlist, dmu_tx_t *tx) { dnode_t *dn; @@ -1257,18 +1331,120 @@ dmu_objset_userused_enabled(objset_t *os) DMU_USERUSED_DNODE(os) != NULL); } +boolean_t +dmu_objset_userobjused_enabled(objset_t *os) +{ + return (dmu_objset_userused_enabled(os) && + spa_feature_is_enabled(os->os_spa, SPA_FEATURE_USEROBJ_ACCOUNTING)); +} + +typedef struct userquota_node { + /* must be in the first filed, see userquota_update_cache() */ + char uqn_id[20 + DMU_OBJACCT_PREFIX_LEN]; + int64_t uqn_delta; + avl_node_t uqn_node; +} userquota_node_t; + +typedef struct userquota_cache { + avl_tree_t uqc_user_deltas; + avl_tree_t uqc_group_deltas; +} userquota_cache_t; + +static int +userquota_compare(const void *l, const void *r) +{ + const userquota_node_t *luqn = l; + const userquota_node_t *ruqn = r; + + /* + * NB: can only access uqn_id because userquota_update_cache() doesn't + * pass in an entire userquota_node_t. + */ + return (strcmp(luqn->uqn_id, ruqn->uqn_id)); +} + static void -do_userquota_update(objset_t *os, uint64_t used, uint64_t flags, - uint64_t user, uint64_t group, boolean_t subtract, dmu_tx_t *tx) +do_userquota_cacheflush(objset_t *os, userquota_cache_t *cache, dmu_tx_t *tx) +{ + void *cookie; + userquota_node_t *uqn; + + ASSERT(dmu_tx_is_syncing(tx)); + + cookie = NULL; + while ((uqn = avl_destroy_nodes(&cache->uqc_user_deltas, + &cookie)) != NULL) { + VERIFY0(zap_increment(os, DMU_USERUSED_OBJECT, + uqn->uqn_id, uqn->uqn_delta, tx)); + kmem_free(uqn, sizeof (*uqn)); + } + avl_destroy(&cache->uqc_user_deltas); + + cookie = NULL; + while ((uqn = avl_destroy_nodes(&cache->uqc_group_deltas, + &cookie)) != NULL) { + VERIFY0(zap_increment(os, DMU_GROUPUSED_OBJECT, + uqn->uqn_id, uqn->uqn_delta, tx)); + kmem_free(uqn, sizeof (*uqn)); + } + avl_destroy(&cache->uqc_group_deltas); +} + +static void +userquota_update_cache(avl_tree_t *avl, const char *id, int64_t delta) +{ + userquota_node_t *uqn; + avl_index_t idx; + + ASSERT(strlen(id) < sizeof (uqn->uqn_id)); + /* + * Use id directly for searching because uqn_id is the first field of + * userquota_node_t and fields after uqn_id won't be accessed in + * avl_find(). + */ + uqn = avl_find(avl, (const void *)id, &idx); + if (uqn == NULL) { + uqn = kmem_zalloc(sizeof (*uqn), KM_SLEEP); + strcpy(uqn->uqn_id, id); + avl_insert(avl, uqn, idx); + } + uqn->uqn_delta += delta; +} + +static void +do_userquota_update(userquota_cache_t *cache, uint64_t used, uint64_t flags, + uint64_t user, uint64_t group, boolean_t subtract) { if ((flags & DNODE_FLAG_USERUSED_ACCOUNTED)) { int64_t delta = DNODE_MIN_SIZE + used; + char name[20]; + if (subtract) delta = -delta; - VERIFY3U(0, ==, zap_increment_int(os, DMU_USERUSED_OBJECT, - user, delta, tx)); - VERIFY3U(0, ==, zap_increment_int(os, DMU_GROUPUSED_OBJECT, - group, delta, tx)); + + (void) sprintf(name, "%llx", (longlong_t)user); + userquota_update_cache(&cache->uqc_user_deltas, name, delta); + + (void) sprintf(name, "%llx", (longlong_t)group); + userquota_update_cache(&cache->uqc_group_deltas, name, delta); + } +} + +static void +do_userobjquota_update(userquota_cache_t *cache, uint64_t flags, + uint64_t user, uint64_t group, boolean_t subtract) +{ + if (flags & DNODE_FLAG_USEROBJUSED_ACCOUNTED) { + char name[20 + DMU_OBJACCT_PREFIX_LEN]; + int delta = subtract ? -1 : 1; + + (void) snprintf(name, sizeof (name), DMU_OBJACCT_PREFIX "%llx", + (longlong_t)user); + userquota_update_cache(&cache->uqc_user_deltas, name, delta); + + (void) snprintf(name, sizeof (name), DMU_OBJACCT_PREFIX "%llx", + (longlong_t)group); + userquota_update_cache(&cache->uqc_group_deltas, name, delta); } } @@ -1277,9 +1453,15 @@ dmu_objset_do_userquota_updates(objset_t *os, dmu_tx_t *tx) { dnode_t *dn; list_t *list = &os->os_synced_dnodes; + userquota_cache_t cache = { { 0 } }; ASSERT(list_head(list) == NULL || dmu_objset_userused_enabled(os)); + avl_create(&cache.uqc_user_deltas, userquota_compare, + sizeof (userquota_node_t), offsetof(userquota_node_t, uqn_node)); + avl_create(&cache.uqc_group_deltas, userquota_compare, + sizeof (userquota_node_t), offsetof(userquota_node_t, uqn_node)); + while ((dn = list_head(list))) { int flags; ASSERT(!DMU_OBJECT_IS_SPECIAL(dn->dn_object)); @@ -1289,32 +1471,27 @@ dmu_objset_do_userquota_updates(objset_t *os, dmu_tx_t *tx) /* Allocate the user/groupused objects if necessary. */ if (DMU_USERUSED_DNODE(os)->dn_type == DMU_OT_NONE) { - VERIFY(0 == zap_create_claim(os, - DMU_USERUSED_OBJECT, + VERIFY0(zap_create_claim(os, DMU_USERUSED_OBJECT, DMU_OT_USERGROUP_USED, DMU_OT_NONE, 0, tx)); - VERIFY(0 == zap_create_claim(os, - DMU_GROUPUSED_OBJECT, + VERIFY0(zap_create_claim(os, DMU_GROUPUSED_OBJECT, DMU_OT_USERGROUP_USED, DMU_OT_NONE, 0, tx)); } - /* - * We intentionally modify the zap object even if the - * net delta is zero. Otherwise - * the block of the zap obj could be shared between - * datasets but need to be different between them after - * a bprewrite. - */ - flags = dn->dn_id_flags; ASSERT(flags); if (flags & DN_ID_OLD_EXIST) { - do_userquota_update(os, dn->dn_oldused, dn->dn_oldflags, - dn->dn_olduid, dn->dn_oldgid, B_TRUE, tx); + do_userquota_update(&cache, + dn->dn_oldused, dn->dn_oldflags, + dn->dn_olduid, dn->dn_oldgid, B_TRUE); + do_userobjquota_update(&cache, dn->dn_oldflags, + dn->dn_olduid, dn->dn_oldgid, B_TRUE); } if (flags & DN_ID_NEW_EXIST) { - do_userquota_update(os, DN_USED_BYTES(dn->dn_phys), - dn->dn_phys->dn_flags, dn->dn_newuid, - dn->dn_newgid, B_FALSE, tx); + do_userquota_update(&cache, + DN_USED_BYTES(dn->dn_phys), dn->dn_phys->dn_flags, + dn->dn_newuid, dn->dn_newgid, B_FALSE); + do_userobjquota_update(&cache, dn->dn_phys->dn_flags, + dn->dn_newuid, dn->dn_newgid, B_FALSE); } mutex_enter(&dn->dn_mtx); @@ -1335,6 +1512,7 @@ dmu_objset_do_userquota_updates(objset_t *os, dmu_tx_t *tx) list_remove(list, dn); dnode_rele(dn, list); } + do_userquota_cacheflush(os, &cache, tx); } /* @@ -1486,19 +1664,19 @@ dmu_objset_userspace_present(objset_t *os) OBJSET_FLAG_USERACCOUNTING_COMPLETE); } -int -dmu_objset_userspace_upgrade(objset_t *os) +boolean_t +dmu_objset_userobjspace_present(objset_t *os) +{ + return (os->os_phys->os_flags & + OBJSET_FLAG_USEROBJACCOUNTING_COMPLETE); +} + +static int +dmu_objset_space_upgrade(objset_t *os) { uint64_t obj; int err = 0; - if (dmu_objset_userspace_present(os)) - return (0); - if (!dmu_objset_userused_enabled(os)) - return (SET_ERROR(ENOTSUP)); - if (dmu_objset_is_snapshot(os)) - return (SET_ERROR(EINVAL)); - /* * We simply need to mark every object dirty, so that it will be * synced out and now accounted. If this is called @@ -1512,6 +1690,13 @@ dmu_objset_userspace_upgrade(objset_t *os) dmu_buf_t *db; int objerr; + mutex_enter(&os->os_upgrade_lock); + if (os->os_upgrade_exit) + err = SET_ERROR(EINTR); + mutex_exit(&os->os_upgrade_lock); + if (err != 0) + return (err); + if (issig(JUSTLOOKING) && issig(FORREAL)) return (SET_ERROR(EINTR)); @@ -1529,12 +1714,60 @@ dmu_objset_userspace_upgrade(objset_t *os) dmu_buf_rele(db, FTAG); dmu_tx_commit(tx); } + return (0); +} + +int +dmu_objset_userspace_upgrade(objset_t *os) +{ + int err = 0; + + if (dmu_objset_userspace_present(os)) + return (0); + if (dmu_objset_is_snapshot(os)) + return (SET_ERROR(EINVAL)); + if (!dmu_objset_userused_enabled(os)) + return (SET_ERROR(ENOTSUP)); + + err = dmu_objset_space_upgrade(os); + if (err) + return (err); os->os_flags |= OBJSET_FLAG_USERACCOUNTING_COMPLETE; txg_wait_synced(dmu_objset_pool(os), 0); return (0); } +static int +dmu_objset_userobjspace_upgrade_cb(objset_t *os) +{ + int err = 0; + + if (dmu_objset_userobjspace_present(os)) + return (0); + if (dmu_objset_is_snapshot(os)) + return (SET_ERROR(EINVAL)); + if (!dmu_objset_userobjused_enabled(os)) + return (SET_ERROR(ENOTSUP)); + + dmu_objset_ds(os)->ds_feature_activation_needed[ + SPA_FEATURE_USEROBJ_ACCOUNTING] = B_TRUE; + + err = dmu_objset_space_upgrade(os); + if (err) + return (err); + + os->os_flags |= OBJSET_FLAG_USEROBJACCOUNTING_COMPLETE; + txg_wait_synced(dmu_objset_pool(os), 0); + return (0); +} + +void +dmu_objset_userobjspace_upgrade(objset_t *os) +{ + dmu_objset_upgrade(os, dmu_objset_userobjspace_upgrade_cb); +} + void dmu_objset_space(objset_t *os, uint64_t *refdbytesp, uint64_t *availbytesp, uint64_t *usedobjsp, uint64_t *availobjsp) @@ -2096,4 +2329,7 @@ EXPORT_SYMBOL(dmu_objset_userquota_get_ids); EXPORT_SYMBOL(dmu_objset_userused_enabled); EXPORT_SYMBOL(dmu_objset_userspace_upgrade); EXPORT_SYMBOL(dmu_objset_userspace_present); +EXPORT_SYMBOL(dmu_objset_userobjused_enabled); +EXPORT_SYMBOL(dmu_objset_userobjspace_upgrade); +EXPORT_SYMBOL(dmu_objset_userobjspace_present); #endif diff --git a/module/zfs/dnode.c b/module/zfs/dnode.c index f0b03bbba..6ba8207e2 100644 --- a/module/zfs/dnode.c +++ b/module/zfs/dnode.c @@ -2001,9 +2001,6 @@ dnode_next_offset_level(dnode_t *dn, int flags, uint64_t *offset, boolean_t hole; int i, inc, error, span; - dprintf("probing object %llu offset %llx level %d of %u\n", - dn->dn_object, *offset, lvl, dn->dn_phys->dn_nlevels); - hole = ((flags & DNODE_FIND_HOLE) != 0); inc = (flags & DNODE_FIND_BACKWARDS) ? -1 : 1; ASSERT(txg == 0 || !hole); diff --git a/module/zfs/dnode_sync.c b/module/zfs/dnode_sync.c index b19f50af9..6d1fa3339 100644 --- a/module/zfs/dnode_sync.c +++ b/module/zfs/dnode_sync.c @@ -570,12 +570,17 @@ dnode_sync(dnode_t *dn, dmu_tx_t *tx) dn->dn_oldused = DN_USED_BYTES(dn->dn_phys); dn->dn_oldflags = dn->dn_phys->dn_flags; dn->dn_phys->dn_flags |= DNODE_FLAG_USERUSED_ACCOUNTED; + if (dmu_objset_userobjused_enabled(dn->dn_objset)) + dn->dn_phys->dn_flags |= + DNODE_FLAG_USEROBJUSED_ACCOUNTED; mutex_exit(&dn->dn_mtx); dmu_objset_userquota_get_ids(dn, B_FALSE, tx); } else { /* Once we account for it, we should always account for it. */ ASSERT(!(dn->dn_phys->dn_flags & DNODE_FLAG_USERUSED_ACCOUNTED)); + ASSERT(!(dn->dn_phys->dn_flags & + DNODE_FLAG_USEROBJUSED_ACCOUNTED)); } mutex_enter(&dn->dn_mtx); diff --git a/module/zfs/spa.c b/module/zfs/spa.c index c2f914e11..0a480d3ec 100644 --- a/module/zfs/spa.c +++ b/module/zfs/spa.c @@ -1167,6 +1167,13 @@ spa_activate(spa_t *spa, int mode) */ spa->spa_zvol_taskq = taskq_create("z_zvol", 1, defclsyspri, 1, INT_MAX, 0); + + /* + * The taskq to upgrade datasets in this pool. Currently used by + * feature SPA_FEATURE_USEROBJ_ACCOUNTING. + */ + spa->spa_upgrade_taskq = taskq_create("z_upgrade", boot_ncpus, + defclsyspri, 1, INT_MAX, TASKQ_DYNAMIC); } /* @@ -1190,6 +1197,11 @@ spa_deactivate(spa_t *spa) spa->spa_zvol_taskq = NULL; } + if (spa->spa_upgrade_taskq) { + taskq_destroy(spa->spa_upgrade_taskq); + spa->spa_upgrade_taskq = NULL; + } + txg_list_destroy(&spa->spa_vdev_txg_list); list_destroy(&spa->spa_config_dirty_list); diff --git a/module/zfs/zfeature_common.c b/module/zfs/zfeature_common.c index 9beb4903e..ccd65a7b7 100644 --- a/module/zfs/zfeature_common.c +++ b/module/zfs/zfeature_common.c @@ -285,4 +285,15 @@ zpool_feature_init(void) "Edon-R hash algorithm.", ZFEATURE_FLAG_PER_DATASET, edonr_deps); } + { + static const spa_feature_t userobj_accounting_deps[] = { + SPA_FEATURE_EXTENSIBLE_DATASET, + SPA_FEATURE_NONE + }; + zfeature_register(SPA_FEATURE_USEROBJ_ACCOUNTING, + "org.zfsonlinux:userobj_accounting", "userobj_accounting", + "User/Group object accounting.", + ZFEATURE_FLAG_READONLY_COMPAT | ZFEATURE_FLAG_PER_DATASET, + userobj_accounting_deps); + } } diff --git a/module/zfs/zfs_acl.c b/module/zfs/zfs_acl.c index 451000010..7198c7ebf 100644 --- a/module/zfs/zfs_acl.c +++ b/module/zfs/zfs_acl.c @@ -1886,7 +1886,9 @@ boolean_t zfs_acl_ids_overquota(zfs_sb_t *zsb, zfs_acl_ids_t *acl_ids) { return (zfs_fuid_overquota(zsb, B_FALSE, acl_ids->z_fuid) || - zfs_fuid_overquota(zsb, B_TRUE, acl_ids->z_fgid)); + zfs_fuid_overquota(zsb, B_TRUE, acl_ids->z_fgid) || + zfs_fuid_overobjquota(zsb, B_FALSE, acl_ids->z_fuid) || + zfs_fuid_overobjquota(zsb, B_TRUE, acl_ids->z_fgid)); } /* diff --git a/module/zfs/zfs_ioctl.c b/module/zfs/zfs_ioctl.c index e5704e258..549a83116 100644 --- a/module/zfs/zfs_ioctl.c +++ b/module/zfs/zfs_ioctl.c @@ -244,9 +244,14 @@ static const char *userquota_perms[] = { ZFS_DELEG_PERM_USERQUOTA, ZFS_DELEG_PERM_GROUPUSED, ZFS_DELEG_PERM_GROUPQUOTA, + ZFS_DELEG_PERM_USEROBJUSED, + ZFS_DELEG_PERM_USEROBJQUOTA, + ZFS_DELEG_PERM_GROUPOBJUSED, + ZFS_DELEG_PERM_GROUPOBJQUOTA, }; static int zfs_ioc_userspace_upgrade(zfs_cmd_t *zc); +static int zfs_ioc_userobjspace_upgrade(zfs_cmd_t *zc); static int zfs_check_settable(const char *name, nvpair_t *property, cred_t *cr); static int zfs_check_clearable(char *dataset, nvlist_t *props, @@ -1171,7 +1176,9 @@ zfs_secpolicy_userspace_one(zfs_cmd_t *zc, nvlist_t *innvl, cred_t *cr) * themself, allow it. */ if (zc->zc_objset_type == ZFS_PROP_USERUSED || - zc->zc_objset_type == ZFS_PROP_USERQUOTA) { + zc->zc_objset_type == ZFS_PROP_USERQUOTA || + zc->zc_objset_type == ZFS_PROP_USEROBJUSED || + zc->zc_objset_type == ZFS_PROP_USEROBJQUOTA) { if (zc->zc_guid == crgetuid(cr)) return (0); } else { @@ -2426,6 +2433,7 @@ zfs_prop_set_special(const char *dsname, zprop_source_t source, zc = kmem_zalloc(sizeof (zfs_cmd_t), KM_SLEEP); (void) strcpy(zc->zc_name, dsname); (void) zfs_ioc_userspace_upgrade(zc); + (void) zfs_ioc_userobjspace_upgrade(zc); kmem_free(zc, sizeof (zfs_cmd_t)); } break; @@ -3720,13 +3728,23 @@ zfs_check_settable(const char *dsname, nvpair_t *pair, cred_t *cr) zfs_userquota_prop_prefixes[ZFS_PROP_USERQUOTA]; const char *gq_prefix = zfs_userquota_prop_prefixes[ZFS_PROP_GROUPQUOTA]; + const char *uiq_prefix = + zfs_userquota_prop_prefixes[ZFS_PROP_USEROBJQUOTA]; + const char *giq_prefix = + zfs_userquota_prop_prefixes[ZFS_PROP_GROUPOBJQUOTA]; if (strncmp(propname, uq_prefix, strlen(uq_prefix)) == 0) { perm = ZFS_DELEG_PERM_USERQUOTA; + } else if (strncmp(propname, uiq_prefix, + strlen(uiq_prefix)) == 0) { + perm = ZFS_DELEG_PERM_USEROBJQUOTA; } else if (strncmp(propname, gq_prefix, strlen(gq_prefix)) == 0) { perm = ZFS_DELEG_PERM_GROUPQUOTA; + } else if (strncmp(propname, giq_prefix, + strlen(giq_prefix)) == 0) { + perm = ZFS_DELEG_PERM_GROUPOBJQUOTA; } else { /* USERUSED and GROUPUSED are read-only */ return (SET_ERROR(EINVAL)); @@ -4927,6 +4945,48 @@ zfs_ioc_userspace_upgrade(zfs_cmd_t *zc) return (error); } +/* + * inputs: + * zc_name name of filesystem + * + * outputs: + * none + */ +static int +zfs_ioc_userobjspace_upgrade(zfs_cmd_t *zc) +{ + objset_t *os; + int error; + + error = dmu_objset_hold(zc->zc_name, FTAG, &os); + if (error != 0) + return (error); + + dsl_dataset_long_hold(dmu_objset_ds(os), FTAG); + dsl_pool_rele(dmu_objset_pool(os), FTAG); + + if (dmu_objset_userobjspace_upgradable(os)) { + mutex_enter(&os->os_upgrade_lock); + if (os->os_upgrade_id == 0) { + /* clear potential error code and retry */ + os->os_upgrade_status = 0; + mutex_exit(&os->os_upgrade_lock); + + dmu_objset_userobjspace_upgrade(os); + } else { + mutex_exit(&os->os_upgrade_lock); + } + + taskq_wait_id(os->os_spa->spa_upgrade_taskq, os->os_upgrade_id); + error = os->os_upgrade_status; + } + + dsl_dataset_long_rele(dmu_objset_ds(os), FTAG); + dsl_dataset_rele(dmu_objset_ds(os), FTAG); + + return (error); +} + static int zfs_ioc_share(zfs_cmd_t *zc) { diff --git a/module/zfs/zfs_vfsops.c b/module/zfs/zfs_vfsops.c index d8b27461a..63b7f9230 100644 --- a/module/zfs/zfs_vfsops.c +++ b/module/zfs/zfs_vfsops.c @@ -431,17 +431,22 @@ zfs_userquota_prop_to_obj(zfs_sb_t *zsb, zfs_userquota_prop_t type) { switch (type) { case ZFS_PROP_USERUSED: + case ZFS_PROP_USEROBJUSED: return (DMU_USERUSED_OBJECT); case ZFS_PROP_GROUPUSED: + case ZFS_PROP_GROUPOBJUSED: return (DMU_GROUPUSED_OBJECT); case ZFS_PROP_USERQUOTA: return (zsb->z_userquota_obj); case ZFS_PROP_GROUPQUOTA: return (zsb->z_groupquota_obj); + case ZFS_PROP_USEROBJQUOTA: + return (zsb->z_userobjquota_obj); + case ZFS_PROP_GROUPOBJQUOTA: + return (zsb->z_groupobjquota_obj); default: - return (SET_ERROR(ENOTSUP)); + return (ZFS_NO_OBJECT); } - return (0); } int @@ -453,16 +458,25 @@ zfs_userspace_many(zfs_sb_t *zsb, zfs_userquota_prop_t type, zap_attribute_t za; zfs_useracct_t *buf = vbuf; uint64_t obj; + int offset = 0; if (!dmu_objset_userspace_present(zsb->z_os)) return (SET_ERROR(ENOTSUP)); + if ((type == ZFS_PROP_USEROBJUSED || type == ZFS_PROP_GROUPOBJUSED || + type == ZFS_PROP_USEROBJQUOTA || type == ZFS_PROP_GROUPOBJQUOTA) && + !dmu_objset_userobjspace_present(zsb->z_os)) + return (SET_ERROR(ENOTSUP)); + obj = zfs_userquota_prop_to_obj(zsb, type); - if (obj == 0) { + if (obj == ZFS_NO_OBJECT) { *bufsizep = 0; return (0); } + if (type == ZFS_PROP_USEROBJUSED || type == ZFS_PROP_GROUPOBJUSED) + offset = DMU_OBJACCT_PREFIX_LEN; + for (zap_cursor_init_serialized(&zc, zsb->z_os, obj, *cookiep); (error = zap_cursor_retrieve(&zc, &za)) == 0; zap_cursor_advance(&zc)) { @@ -470,7 +484,15 @@ zfs_userspace_many(zfs_sb_t *zsb, zfs_userquota_prop_t type, *bufsizep) break; - fuidstr_to_sid(zsb, za.za_name, + /* + * skip object quota (with zap name prefix DMU_OBJACCT_PREFIX) + * when dealing with block quota and vice versa. + */ + if ((offset > 0) != (strncmp(za.za_name, DMU_OBJACCT_PREFIX, + DMU_OBJACCT_PREFIX_LEN) == 0)) + continue; + + fuidstr_to_sid(zsb, za.za_name + offset, buf->zu_domain, sizeof (buf->zu_domain), &buf->zu_rid); buf->zu_space = za.za_first_integer; @@ -511,7 +533,8 @@ int zfs_userspace_one(zfs_sb_t *zsb, zfs_userquota_prop_t type, const char *domain, uint64_t rid, uint64_t *valp) { - char buf[32]; + char buf[20 + DMU_OBJACCT_PREFIX_LEN]; + int offset = 0; int err; uint64_t obj; @@ -520,11 +543,21 @@ zfs_userspace_one(zfs_sb_t *zsb, zfs_userquota_prop_t type, if (!dmu_objset_userspace_present(zsb->z_os)) return (SET_ERROR(ENOTSUP)); + if ((type == ZFS_PROP_USEROBJUSED || type == ZFS_PROP_GROUPOBJUSED || + type == ZFS_PROP_USEROBJQUOTA || type == ZFS_PROP_GROUPOBJQUOTA) && + !dmu_objset_userobjspace_present(zsb->z_os)) + return (SET_ERROR(ENOTSUP)); + obj = zfs_userquota_prop_to_obj(zsb, type); - if (obj == 0) + if (obj == ZFS_NO_OBJECT) return (0); - err = id_to_fuidstr(zsb, domain, rid, buf, B_FALSE); + if (type == ZFS_PROP_USEROBJUSED || type == ZFS_PROP_GROUPOBJUSED) { + strncpy(buf, DMU_OBJACCT_PREFIX, DMU_OBJACCT_PREFIX_LEN); + offset = DMU_OBJACCT_PREFIX_LEN; + } + + err = id_to_fuidstr(zsb, domain, rid, buf + offset, B_FALSE); if (err) return (err); @@ -545,14 +578,25 @@ zfs_set_userquota(zfs_sb_t *zsb, zfs_userquota_prop_t type, uint64_t *objp; boolean_t fuid_dirtied; - if (type != ZFS_PROP_USERQUOTA && type != ZFS_PROP_GROUPQUOTA) - return (SET_ERROR(EINVAL)); - if (zsb->z_version < ZPL_VERSION_USERSPACE) return (SET_ERROR(ENOTSUP)); - objp = (type == ZFS_PROP_USERQUOTA) ? &zsb->z_userquota_obj : - &zsb->z_groupquota_obj; + switch (type) { + case ZFS_PROP_USERQUOTA: + objp = &zsb->z_userquota_obj; + break; + case ZFS_PROP_GROUPQUOTA: + objp = &zsb->z_groupquota_obj; + break; + case ZFS_PROP_USEROBJQUOTA: + objp = &zsb->z_userobjquota_obj; + break; + case ZFS_PROP_GROUPOBJQUOTA: + objp = &zsb->z_groupobjquota_obj; + break; + default: + return (SET_ERROR(EINVAL)); + } err = id_to_fuidstr(zsb, domain, rid, buf, B_TRUE); if (err) @@ -598,9 +642,39 @@ zfs_set_userquota(zfs_sb_t *zsb, zfs_userquota_prop_t type, EXPORT_SYMBOL(zfs_set_userquota); boolean_t +zfs_fuid_overobjquota(zfs_sb_t *zsb, boolean_t isgroup, uint64_t fuid) +{ + char buf[20 + DMU_OBJACCT_PREFIX_LEN]; + uint64_t used, quota, usedobj, quotaobj; + int err; + + if (!dmu_objset_userobjspace_present(zsb->z_os)) { + if (dmu_objset_userobjspace_upgradable(zsb->z_os)) + dmu_objset_userobjspace_upgrade(zsb->z_os); + return (B_FALSE); + } + + usedobj = isgroup ? DMU_GROUPUSED_OBJECT : DMU_USERUSED_OBJECT; + quotaobj = isgroup ? zsb->z_groupobjquota_obj : zsb->z_userobjquota_obj; + if (quotaobj == 0 || zsb->z_replay) + return (B_FALSE); + + (void) sprintf(buf, "%llx", (longlong_t)fuid); + err = zap_lookup(zsb->z_os, quotaobj, buf, 8, 1, "a); + if (err != 0) + return (B_FALSE); + + (void) sprintf(buf, DMU_OBJACCT_PREFIX "%llx", (longlong_t)fuid); + err = zap_lookup(zsb->z_os, usedobj, buf, 8, 1, &used); + if (err != 0) + return (B_FALSE); + return (used >= quota); +} + +boolean_t zfs_fuid_overquota(zfs_sb_t *zsb, boolean_t isgroup, uint64_t fuid) { - char buf[32]; + char buf[20]; uint64_t used, quota, usedobj, quotaobj; int err; @@ -777,6 +851,18 @@ zfs_sb_create(const char *osname, zfs_mntopts_t *zmo, zfs_sb_t **zsbp) if (error && error != ENOENT) goto out; + error = zap_lookup(os, MASTER_NODE_OBJ, + zfs_userquota_prop_prefixes[ZFS_PROP_USEROBJQUOTA], + 8, 1, &zsb->z_userobjquota_obj); + if (error && error != ENOENT) + goto out; + + error = zap_lookup(os, MASTER_NODE_OBJ, + zfs_userquota_prop_prefixes[ZFS_PROP_GROUPOBJQUOTA], + 8, 1, &zsb->z_groupobjquota_obj); + if (error && error != ENOENT) + goto out; + error = zap_lookup(os, MASTER_NODE_OBJ, ZFS_FUID_TABLES, 8, 1, &zsb->z_fuid_obj); if (error && error != ENOENT) diff --git a/module/zfs/zfs_znode.c b/module/zfs/zfs_znode.c index 647ce48dc..11bb8675b 100644 --- a/module/zfs/zfs_znode.c +++ b/module/zfs/zfs_znode.c @@ -478,25 +478,6 @@ zfs_inode_set_ops(zfs_sb_t *zsb, struct inode *ip) } } -void -zfs_set_inode_flags(znode_t *zp, struct inode *ip) -{ - /* - * Linux and Solaris have different sets of file attributes, so we - * restrict this conversion to the intersection of the two. - */ - - if (zp->z_pflags & ZFS_IMMUTABLE) - ip->i_flags |= S_IMMUTABLE; - else - ip->i_flags &= ~S_IMMUTABLE; - - if (zp->z_pflags & ZFS_APPENDONLY) - ip->i_flags |= S_APPEND; - else - ip->i_flags &= ~S_APPEND; -} - /* * Update the embedded inode given the znode. We should work toward * eliminating this function as soon as possible by removing values @@ -523,7 +504,6 @@ zfs_inode_update(znode_t *zp) dmu_object_size_from_db(sa_get_db(zp->z_sa_hdl), &blksize, &i_blocks); spin_lock(&ip->i_lock); - zfs_set_inode_flags(zp, ip); ip->i_blocks = i_blocks; i_size_write(ip, zp->z_size); spin_unlock(&ip->i_lock); @@ -946,6 +926,7 @@ zfs_xvattr_set(znode_t *zp, xvattr_t *xvap, dmu_tx_t *tx) ×, sizeof (times), tx); XVA_SET_RTN(xvap, XAT_CREATETIME); } + if (XVA_ISSET_REQ(xvap, XAT_READONLY)) { ZFS_ATTR_SET(zp, ZFS_READONLY, xoap->xoa_readonly, zp->z_pflags, tx); @@ -970,7 +951,12 @@ zfs_xvattr_set(znode_t *zp, xvattr_t *xvap, dmu_tx_t *tx) ZFS_ATTR_SET(zp, ZFS_IMMUTABLE, xoap->xoa_immutable, zp->z_pflags, tx); XVA_SET_RTN(xvap, XAT_IMMUTABLE); + + ZTOI(zp)->i_flags |= S_IMMUTABLE; + } else { + ZTOI(zp)->i_flags &= ~S_IMMUTABLE; } + if (XVA_ISSET_REQ(xvap, XAT_NOUNLINK)) { ZFS_ATTR_SET(zp, ZFS_NOUNLINK, xoap->xoa_nounlink, zp->z_pflags, tx); @@ -980,7 +966,13 @@ zfs_xvattr_set(znode_t *zp, xvattr_t *xvap, dmu_tx_t *tx) ZFS_ATTR_SET(zp, ZFS_APPENDONLY, xoap->xoa_appendonly, zp->z_pflags, tx); XVA_SET_RTN(xvap, XAT_APPENDONLY); + + ZTOI(zp)->i_flags |= S_APPEND; + } else { + + ZTOI(zp)->i_flags &= ~S_APPEND; } + if (XVA_ISSET_REQ(xvap, XAT_NODUMP)) { ZFS_ATTR_SET(zp, ZFS_NODUMP, xoap->xoa_nodump, zp->z_pflags, tx); @@ -1234,6 +1226,7 @@ zfs_rezget(znode_t *zp) zp->z_atime_dirty = 0; zfs_inode_update(zp); + zfs_znode_hold_exit(zsb, zh); return (0); diff --git a/scripts/dkms.mkconf b/scripts/dkms.mkconf index caba1becd..ad97f9bca 100755 --- a/scripts/dkms.mkconf +++ b/scripts/dkms.mkconf @@ -70,6 +70,7 @@ STRIP[2]="\${STRIP[0]}" STRIP[3]="\${STRIP[0]}" STRIP[4]="\${STRIP[0]}" STRIP[5]="\${STRIP[0]}" +STRIP[6]="\${STRIP[0]}" BUILT_MODULE_NAME[0]="zavl" BUILT_MODULE_LOCATION[0]="module/avl/" DEST_MODULE_LOCATION[0]="/extra/avl/avl" @@ -88,4 +89,7 @@ DEST_MODULE_LOCATION[4]="/extra/zfs/zfs" BUILT_MODULE_NAME[5]="zpios" BUILT_MODULE_LOCATION[5]="module/zpios/" DEST_MODULE_LOCATION[5]="/extra/zpios/zpios" +BUILT_MODULE_NAME[6]="icp" +BUILT_MODULE_LOCATION[6]="module/icp/" +DEST_MODULE_LOCATION[6]="/extra/icp/icp" EOF diff --git a/tests/runfiles/linux.run b/tests/runfiles/linux.run index 92f867ab9..bc2373b70 100644 --- a/tests/runfiles/linux.run +++ b/tests/runfiles/linux.run @@ -144,7 +144,7 @@ tests = [] tests = ['zfs_receive_001_pos', 'zfs_receive_002_pos', 'zfs_receive_003_pos', 'zfs_receive_005_neg', 'zfs_receive_006_pos', 'zfs_receive_007_neg', 'zfs_receive_008_pos', 'zfs_receive_009_neg', - 'zfs_receive_010_pos', 'zfs_receive_012_pos'] + 'zfs_receive_010_pos', 'zfs_receive_012_pos', 'zfs_receive_013_pos'] # DISABLED: # zfs_rename_002_pos - needs investigation @@ -355,7 +355,8 @@ tests = ['zdb_001_neg', 'zfs_001_neg', 'zfs_allow_001_neg', 'zpool_history_001_neg', 'zpool_import_001_neg', 'zpool_import_002_neg', 'zpool_offline_001_neg', 'zpool_online_001_neg', 'zpool_remove_001_neg', 'zpool_replace_001_neg', 'zpool_scrub_001_neg', 'zpool_set_001_neg', - 'zpool_status_001_neg', 'zpool_upgrade_001_neg'] + 'zpool_status_001_neg', 'zpool_upgrade_001_neg', 'arcstat_001_pos', + 'arc_summary_001_pos', 'dbufstat_001_pos'] user = [tests/functional/cli_user/zfs_list] @@ -602,21 +603,18 @@ tests = ['sparse_001_pos'] [tests/functional/truncate] tests = ['truncate_001_pos', 'truncate_002_pos'] -# DISABLED: -# groupspace_001_pos -# groupspace_002_pos -# userquota_001_pos -# userquota_004_pos -# userquota_007_pos -# userquota_010_pos -# userspace_001_pos -# userspace_002_pos +[tests/functional/upgrade] +tests = [ 'upgrade_userobj_001_pos' ] + [tests/functional/userquota] tests = [ - 'userquota_002_pos', 'userquota_003_pos', - 'userquota_005_neg', 'userquota_006_pos', - 'userquota_008_pos', 'userquota_009_pos', - 'userquota_011_pos', 'userquota_012_neg'] + 'userquota_001_pos', 'userquota_002_pos', 'userquota_003_pos', + 'userquota_004_pos', 'userquota_005_neg', 'userquota_006_pos', + 'userquota_007_pos', 'userquota_008_pos', 'userquota_009_pos', + 'userquota_010_pos', 'userquota_011_pos', 'userquota_012_neg', + 'userquota_013_pos', + 'userspace_001_pos', 'userspace_002_pos', 'userspace_003_pos', + 'groupspace_001_pos', 'groupspace_002_pos', 'groupspace_003_pos' ] # DISABLED: # vdev_zaps_007_pos -- fails due to a pre-existing issue with zpool split diff --git a/tests/test-runner/cmd/test-runner.py b/tests/test-runner/cmd/test-runner.py index 4d31ddceb..687284907 100755 --- a/tests/test-runner/cmd/test-runner.py +++ b/tests/test-runner/cmd/test-runner.py @@ -127,12 +127,12 @@ class Cmd(object): self.killed = False self.result = Result() - if self.timeout == None: - self.timeout = 60 + if self.timeout is None: + self.timeout = 60 def __str__(self): return "Pathname: %s\nOutputdir: %s\nTimeout: %d\nUser: %s\n" % ( - self.pathname, self.outputdir, self.timeout, self.user) + self.pathname, self.outputdir, self.timeout, self.user) def kill_cmd(self, proc): """ @@ -309,9 +309,10 @@ class Test(Cmd): if len(self.post_user): post_user = ' (as %s)' % (self.post_user) return "Pathname: %s\nOutputdir: %s\nTimeout: %d\nPre: %s%s\nPost: " \ - "%s%s\nUser: %s\n" % (self.pathname, self.outputdir, - self.timeout, self.pre, pre_user, self.post, post_user, - self.user) + "%s%s\nUser: %s\n" % ( + self.pathname, self.outputdir, + self.timeout, self.pre, pre_user, self.post, post_user, + self.user) def verify(self, logger): """ @@ -384,9 +385,9 @@ class TestGroup(Test): if len(self.post_user): post_user = ' (as %s)' % (self.post_user) return "Pathname: %s\nOutputdir: %s\nTests: %s\nTimeout: %d\n" \ - "Pre: %s%s\nPost: %s%s\nUser: %s\n" % (self.pathname, - self.outputdir, self.tests, self.timeout, self.pre, pre_user, - self.post, post_user, self.user) + "Pre: %s%s\nPost: %s%s\nUser: %s\n" % ( + self.pathname, self.outputdir, self.tests, self.timeout, + self.pre, pre_user, self.post, post_user, self.user) def verify(self, logger): """ @@ -428,8 +429,8 @@ class TestGroup(Test): if not verify_file(os.path.join(self.pathname, test)): del self.tests[self.tests.index(test)] logger.info("Warning: Test '%s' removed from TestGroup '%s' " - "because it failed verification." % (test, - self.pathname)) + "because it failed verification." % + (test, self.pathname)) return len(self.tests) is not 0 @@ -634,7 +635,7 @@ class TestRun(object): components -= 1 for testfile in tmp_dict.keys(): uniq = '/'.join(testfile.split('/')[components:]).lstrip('/') - if not uniq in l: + if uniq not in l: l.append(uniq) tmp_dict[testfile].outputdir = os.path.join(base, uniq) else: @@ -705,8 +706,8 @@ class TestRun(object): m, s = divmod(time() - self.starttime, 60) h, m = divmod(m, 60) print '\nRunning Time:\t%02d:%02d:%02d' % (h, m, s) - print 'Percent passed:\t%.1f%%' % ((float(Result.runresults['PASS']) / - float(Result.total)) * 100) + print 'Percent passed:\t%.1f%%' % ( + (float(Result.runresults['PASS']) / float(Result.total)) * 100) print 'Log directory:\t%s' % self.outputdir @@ -717,10 +718,10 @@ def verify_file(pathname): if os.path.isdir(pathname) or os.path.islink(pathname): return False - if (os.path.isfile(pathname) and os.access(pathname, os.X_OK)) or \ - (os.path.isfile(pathname+'.ksh') and os.access(pathname+'.ksh', os.X_OK)) or \ - (os.path.isfile(pathname+'.sh') and os.access(pathname+'.sh', os.X_OK)): - return True + for ext in '', '.ksh', '.sh': + script_path = pathname + ext + if os.path.isfile(script_path) and os.access(script_path, os.X_OK): + return True return False @@ -731,15 +732,13 @@ def verify_user(user, logger): sudo without being prompted for a password. """ testcmd = [SUDO, '-n', '-u', user, TRUE] - can_sudo = exists = True if user in Cmd.verified_users: return True try: - _ = getpwnam(user) + getpwnam(user) except KeyError: - exists = False logger.info("Warning: user '%s' does not exist.", user) return False @@ -782,7 +781,7 @@ def options_cb(option, opt_str, value, parser): path_options = ['runfile', 'outputdir', 'template', 'testdir'] if option.dest is 'runfile' and '-w' in parser.rargs or \ - option.dest is 'template' and '-c' in parser.rargs: + option.dest is 'template' and '-c' in parser.rargs: fail('-c and -w are mutually exclusive.') if opt_str in parser.rargs: diff --git a/tests/zfs-tests/cmd/mkfiles/mkfiles.c b/tests/zfs-tests/cmd/mkfiles/mkfiles.c index 418fb9d07..62dee1627 100644 --- a/tests/zfs-tests/cmd/mkfiles/mkfiles.c +++ b/tests/zfs-tests/cmd/mkfiles/mkfiles.c @@ -48,10 +48,7 @@ main(int argc, char **argv) if (argc == 4 && sscanf(argv[3], "%u", &first_file) != 1) usage("Invalid first file", -3); - if (numfiles < first_file) - usage("First file larger than last file", -3); - - for (i = first_file; i <= numfiles; i++) { + for (i = first_file; i < first_file + numfiles; i++) { int fd; (void) snprintf(buf, MAXPATHLEN, "%s%u", argv[1], i); if ((fd = open(buf, O_CREAT | O_EXCL, O_RDWR)) == -1) { diff --git a/tests/zfs-tests/include/default.cfg.in b/tests/zfs-tests/include/default.cfg.in index a36ab3b1a..f93bfa98c 100644 --- a/tests/zfs-tests/include/default.cfg.in +++ b/tests/zfs-tests/include/default.cfg.in @@ -43,6 +43,9 @@ export ZPOOL=${ZPOOL:-${sbindir}/zpool} export ZTEST=${ZTEST:-${sbindir}/ztest} export ZPIOS=${ZPIOS:-${sbindir}/zpios} export RAIDZ_TEST=${RAIDZ_TEST:-${bindir}/raidz_test} +export ARC_SUMMARY=${ARC_SUMMARY:-${bindir}/arc_summary.py} +export ARCSTAT=${ARCSTAT:-${bindir}/arcstat.py} +export DBUFSTAT=${DBUFSTAT:-${bindir}/dbufstat.py} . $STF_SUITE/include/libtest.shlib diff --git a/tests/zfs-tests/include/libtest.shlib b/tests/zfs-tests/include/libtest.shlib index 62ba3a9eb..1857cf91f 100644 --- a/tests/zfs-tests/include/libtest.shlib +++ b/tests/zfs-tests/include/libtest.shlib @@ -2857,3 +2857,29 @@ function block_device_wait $UDEVADM settle fi } + +# +# Synchronize all the data in pool +# +# $1 pool name +# +function sync_pool #pool +{ + typeset pool=${1:-$TESTPOOL} + + log_must $SYNC + log_must $SLEEP 2 + # Flush all the pool data. + typeset -i ret + $ZPOOL scrub $pool >/dev/null 2>&1 + ret=$? + (( $ret != 0 )) && \ + log_fail "$ZPOOL scrub $pool failed." + + while ! is_pool_scrubbed $pool; do + if is_pool_resilvered $pool ; then + log_fail "$pool should not be resilver completed." + fi + log_must $SLEEP 2 + done +} diff --git a/tests/zfs-tests/tests/functional/Makefile.am b/tests/zfs-tests/tests/functional/Makefile.am index ed01eafb4..ffba71b51 100644 --- a/tests/zfs-tests/tests/functional/Makefile.am +++ b/tests/zfs-tests/tests/functional/Makefile.am @@ -51,6 +51,7 @@ SUBDIRS = \ sparse \ threadsappend \ truncate \ + upgrade \ userquota \ vdev_zaps \ write_dirs \ diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_receive/Makefile.am b/tests/zfs-tests/tests/functional/cli_root/zfs_receive/Makefile.am index f5857f4a4..8aadbf067 100644 --- a/tests/zfs-tests/tests/functional/cli_root/zfs_receive/Makefile.am +++ b/tests/zfs-tests/tests/functional/cli_root/zfs_receive/Makefile.am @@ -13,4 +13,5 @@ dist_pkgdata_SCRIPTS = \ zfs_receive_009_neg.ksh \ zfs_receive_010_pos.ksh \ zfs_receive_011_pos.ksh \ - zfs_receive_012_pos.ksh + zfs_receive_012_pos.ksh \ + zfs_receive_013_pos.ksh diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_013_pos.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_013_pos.ksh new file mode 100755 index 000000000..11081bb7f --- /dev/null +++ b/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_013_pos.ksh @@ -0,0 +1,83 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright 2016, loli10K. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/cli_root/zfs_set/zfs_set_common.kshlib + +# +# DESCRIPTION: +# Verify ZFS can receive custom properties on both filesystems and +# snapshots from full and incremental streams. +# +# STRATEGY: +# 1. Create a filesystem. +# 2. Snapshot the filesystem. +# 3. Set custom properties on both the fs and snapshots. +# 4. Create different send streams with the properties. +# 5. Receive the send streams and verify the properties. +# + +verify_runnable "both" + +typeset streamfile_full=/var/tmp/streamfile_full.$$ +typeset streamfile_incr=/var/tmp/streamfile_incr.$$ +orig=$TESTPOOL/$TESTFS1 +dest=$TESTPOOL/$TESTFS2 +typeset user_prop=$(valid_user_property 8) +typeset value=$(user_property_value 8) + +function cleanup +{ + log_must $RM $streamfile_full + log_must $RM $streamfile_incr + log_must $ZFS destroy -rf $TESTPOOL/$TESTFS1 + log_must $ZFS destroy -rf $TESTPOOL/$TESTFS2 +} + +log_assert "ZFS can receive custom properties." +log_onexit cleanup + +# 1. Create a filesystem. +log_must $ZFS create $orig + +# 2. Snapshot the filesystem. +log_must $ZFS snapshot $orig@snap1 +log_must $ZFS snapshot $orig@snap2 +log_must $ZFS snapshot $orig@snap3 + +# 3. Set custom properties on both the fs and snapshots. +log_must eval "$ZFS set '$user_prop'='$value' $orig" +log_must eval "$ZFS set '$user_prop:snap1'='$value:snap1' $orig@snap1" +log_must eval "$ZFS set '$user_prop:snap2'='$value:snap2' $orig@snap2" +log_must eval "$ZFS set '$user_prop:snap3'='$value:snap3' $orig@snap3" + +# 4. Create different send streams with the properties. +log_must eval "$ZFS send -p $orig@snap1 > $streamfile_full" +log_must eval "$ZFS send -p -I $orig@snap1 $orig@snap3 > $streamfile_incr" + +# 5. Receive the send streams and verify the properties. +log_must eval "$ZFS recv $dest < $streamfile_full" +log_must eval "check_user_prop $dest $user_prop '$value'" +log_must eval "check_user_prop $dest@snap1 '$user_prop:snap1' '$value:snap1'" +log_must eval "$ZFS recv $dest < $streamfile_incr" +log_must eval "check_user_prop $dest@snap2 '$user_prop:snap2' '$value:snap2'" +log_must eval "check_user_prop $dest@snap3 '$user_prop:snap3' '$value:snap3'" + +log_pass "ZFS can receive custom properties passed." diff --git a/tests/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg b/tests/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg index 3807d0af6..699229fef 100644 --- a/tests/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg +++ b/tests/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg @@ -38,7 +38,8 @@ typeset -a properties=("size" "capacity" "altroot" "health" "guid" "version" "feature@large_blocks" "feature@large_dnode" "feature@filesystem_limits" "feature@spacemap_histogram" "feature@enabled_txg" "feature@hole_birth" "feature@extensible_dataset" "feature@bookmarks" "feature@embedded_data" - "feature@sha512" "feature@skein" "feature@edonr") + "feature@sha512" "feature@skein" "feature@edonr" + "feature@userobj_accounting") else typeset -a properties=("size" "capacity" "altroot" "health" "guid" "version" "bootfs" ""leaked" delegation" "autoreplace" "cachefile" "dedupditto" "dedupratio" diff --git a/tests/zfs-tests/tests/functional/cli_user/misc/Makefile.am b/tests/zfs-tests/tests/functional/cli_user/misc/Makefile.am index dad922c6d..cf7502c27 100644 --- a/tests/zfs-tests/tests/functional/cli_user/misc/Makefile.am +++ b/tests/zfs-tests/tests/functional/cli_user/misc/Makefile.am @@ -43,4 +43,7 @@ dist_pkgdata_SCRIPTS = \ zpool_scrub_001_neg.ksh \ zpool_set_001_neg.ksh \ zpool_status_001_neg.ksh \ - zpool_upgrade_001_neg.ksh + zpool_upgrade_001_neg.ksh \ + arcstat_001_pos.ksh \ + arc_summary_001_pos.ksh \ + dbufstat_001_pos.ksh diff --git a/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary_001_pos.ksh b/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary_001_pos.ksh new file mode 100755 index 000000000..e65fbe6b4 --- /dev/null +++ b/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary_001_pos.ksh @@ -0,0 +1,40 @@ +#! /bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2015 by Lawrence Livermore National Security, LLC. +# All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib + +set -A args "" "-a" "-d" "-p 1" + +log_assert "arc_summary.py generates output and doesn't return an error code" + +typeset -i i=0 +while [[ $i -lt ${#args[*]} ]]; do + log_must eval "$ARC_SUMMARY ${args[i]} > /dev/null" + ((i = i + 1)) +done + +log_pass "arc_summary.py generates output and doesn't return an error code" diff --git a/tests/zfs-tests/tests/functional/cli_user/misc/arcstat_001_pos.ksh b/tests/zfs-tests/tests/functional/cli_user/misc/arcstat_001_pos.ksh new file mode 100755 index 000000000..edf80b10f --- /dev/null +++ b/tests/zfs-tests/tests/functional/cli_user/misc/arcstat_001_pos.ksh @@ -0,0 +1,41 @@ +#! /bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2015 by Lawrence Livermore National Security, LLC. +# All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib + +set -A args "" "-s \",\"" "-x" "-v" \ + "-f time,hit%,dh%,ph%,mh%" + +log_assert "arcstat.py generates output and doesn't return an error code" + +typeset -i i=0 +while [[ $i -lt ${#args[*]} ]]; do + log_must eval "$ARCSTAT ${args[i]} > /dev/null" + ((i = i + 1)) +done +log_pass "arcstat.py generates output and doesn't return an error code" + diff --git a/tests/zfs-tests/tests/functional/cli_user/misc/dbufstat_001_pos.ksh b/tests/zfs-tests/tests/functional/cli_user/misc/dbufstat_001_pos.ksh new file mode 100755 index 000000000..229ba72cb --- /dev/null +++ b/tests/zfs-tests/tests/functional/cli_user/misc/dbufstat_001_pos.ksh @@ -0,0 +1,40 @@ +#! /bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2015 by Lawrence Livermore National Security, LLC. +# All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib + +set -A args "" "-b" "-d" "-r" "-v" "-s \",\"" "-x" + +log_assert "dbufstat.py generates output and doesn't return an error code" + +typeset -i i=0 +while [[ $i -lt ${#args[*]} ]]; do + log_must eval "$DBUFSTAT ${args[i]} > /dev/null" + ((i = i + 1)) +done + +log_pass "dbufstat.py generates output and doesn't return an error code" diff --git a/tests/zfs-tests/tests/functional/redundancy/redundancy.kshlib b/tests/zfs-tests/tests/functional/redundancy/redundancy.kshlib index cb8271797..56e2bd19d 100644 --- a/tests/zfs-tests/tests/functional/redundancy/redundancy.kshlib +++ b/tests/zfs-tests/tests/functional/redundancy/redundancy.kshlib @@ -214,32 +214,6 @@ function get_vdevs #pool cnt } # -# Synchronize all the data in pool -# -# $1 pool name -# -function sync_pool #pool -{ - typeset pool=$1 - - log_must $SYNC - log_must $SLEEP 2 - # Flush all the pool data. - typeset -i ret - $ZPOOL scrub $pool >/dev/null 2>&1 - ret=$? - (( $ret != 0 )) && \ - log_fail "$ZPOOL scrub $pool failed." - - while ! is_pool_scrubbed $pool; do - if is_pool_resilvered $pool ; then - log_fail "$pool should not be resilver completed." - fi - log_must $SLEEP 2 - done -} - -# # Create and replace the same name virtual device files # # $1 pool name diff --git a/tests/zfs-tests/tests/functional/upgrade/Makefile.am b/tests/zfs-tests/tests/functional/upgrade/Makefile.am new file mode 100644 index 000000000..31034342f --- /dev/null +++ b/tests/zfs-tests/tests/functional/upgrade/Makefile.am @@ -0,0 +1,5 @@ +pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/upgrade +dist_pkgdata_SCRIPTS = \ + setup.ksh \ + cleanup.ksh \ + upgrade_userobj_001_pos.ksh diff --git a/tests/zfs-tests/tests/functional/upgrade/cleanup.ksh b/tests/zfs-tests/tests/functional/upgrade/cleanup.ksh new file mode 100644 index 000000000..6b0eb9d9a --- /dev/null +++ b/tests/zfs-tests/tests/functional/upgrade/cleanup.ksh @@ -0,0 +1,44 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +# +# Copyright (c) 2013 by Delphix. All rights reserved. +# + +# +# Copyright (c) 2016 by Jinshan Xiong. No rights reserved. +# + +. $STF_SUITE/include/libtest.shlib + +verify_runnable "global" + +log_must $ZPOOL destroy $TESTPOOL + +log_must $RM /tmp/zpool_upgrade_test.dat + +default_cleanup diff --git a/tests/zfs-tests/tests/functional/upgrade/setup.ksh b/tests/zfs-tests/tests/functional/upgrade/setup.ksh new file mode 100644 index 000000000..57b483581 --- /dev/null +++ b/tests/zfs-tests/tests/functional/upgrade/setup.ksh @@ -0,0 +1,44 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +# +# Copyright (c) 2013 by Delphix. All rights reserved. +# + +# +# Copyright (c) 2016 by Jinshan Xiong. No rights reserved. +# + +. $STF_SUITE/include/libtest.shlib + +verify_runnable "global" + +# create a pool without any features +log_must $MKFILE 128m /tmp/zpool_upgrade_test.dat +log_must $ZPOOL create -d -m $TESTDIR $TESTPOOL /tmp/zpool_upgrade_test.dat + +log_pass diff --git a/tests/zfs-tests/tests/functional/upgrade/upgrade_userobj_001_pos.ksh b/tests/zfs-tests/tests/functional/upgrade/upgrade_userobj_001_pos.ksh new file mode 100644 index 000000000..49087f5a1 --- /dev/null +++ b/tests/zfs-tests/tests/functional/upgrade/upgrade_userobj_001_pos.ksh @@ -0,0 +1,98 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2013 by Jinshan Xiong. No rights reserved. +# + +. $STF_SUITE/include/libtest.shlib + +# +# DESCRIPTION: +# +# Check that zfs upgrade for object count accounting works. +# Since userobjaccounting is a per dataset feature, this test case +# will create multiple dataset and try different upgrade method. +# +# STRATEGY: +# 1. Create a pool with all features disabled +# 2. Create a few dataset for testing +# 3. Make sure automatic upgrade work +# 4. Make sure manual upgrade work +# + +function cleanup +{ + datasetexists $TESTPOOL/fs1 && log_must $ZFS destroy $TESTPOOL/fs1 + datasetexists $TESTPOOL/fs2 && log_must $ZFS destroy $TESTPOOL/fs2 +} + +verify_runnable "global" + +log_assert "pool upgrade for userobj accounting should work" +log_onexit cleanup + +log_must $MKFILES $TESTDIR/tf $((RANDOM % 1000 + 1)) +log_must $ZFS create $TESTPOOL/fs1 +log_must $MKFILES $TESTDIR/fs1/tf $((RANDOM % 1000 + 1)) +log_must $ZFS create $TESTPOOL/fs2 +log_must $MKFILES $TESTDIR/fs2/tf $((RANDOM % 1000 + 1)) +log_must $ZFS umount $TESTPOOL/fs2 + +# Make sure userobj accounting is disabled +$ZFS userspace -o objused -H $TESTPOOL | $HEAD -n 1 | $GREP -q "-" || + log_fail "userobj accounting should be disabled initially" + +# Upgrade zpool to support all features +log_must $ZPOOL upgrade $TESTPOOL + +# Make sure userobj accounting is disabled again +$ZFS userspace -o objused -H $TESTPOOL | $HEAD -n 1 | $GREP -q "-" || + log_fail "userobj accounting should be disabled after pool upgrade" + +# Create a file in fs1 should trigger dataset upgrade +log_must $MKFILE 1m $TESTDIR/fs1/tf +sync_pool + +# Make sure userobj accounting is working for fs1 +$ZFS userspace -o objused -H $TESTPOOL/fs1 | $HEAD -n 1 | $GREP -q "-" && + log_fail "userobj accounting should be enabled for $TESTPOOL/fs1" + +# Mount a dataset should trigger upgrade +log_must $ZFS mount $TESTPOOL/fs2 +sync_pool + +# Make sure userobj accounting is working for fs2 +$ZFS userspace -o objused -H $TESTPOOL/fs2 | $HEAD -n 1 | $GREP -q "-" && + log_fail "userobj accounting should be enabled for $TESTPOOL/fs2" + +# All in all, after having been through this, the dataset for testpool +# still shouldn't be upgraded +$ZFS userspace -o objused -H $TESTPOOL | $HEAD -n 1 | $GREP -q "-" || + log_fail "userobj accounting should be disabled for $TESTPOOL" + +# Manual upgrade root dataset +log_must $ZFS set version=current $TESTPOOL +$ZFS userspace -o objused -H $TESTPOOL | $HEAD -n 1 | $GREP -q "-" && + log_fail "userobj accounting should be enabled for $TESTPOOL" + +log_pass "all tests passed - what a lucky day!" diff --git a/tests/zfs-tests/tests/functional/userquota/Makefile.am b/tests/zfs-tests/tests/functional/userquota/Makefile.am index b72659964..b7f38f98e 100644 --- a/tests/zfs-tests/tests/functional/userquota/Makefile.am +++ b/tests/zfs-tests/tests/functional/userquota/Makefile.am @@ -6,6 +6,7 @@ dist_pkgdata_SCRIPTS = \ cleanup.ksh \ groupspace_001_pos.ksh \ groupspace_002_pos.ksh \ + groupspace_003_pos.ksh \ userquota_001_pos.ksh \ userquota_002_pos.ksh \ userquota_003_pos.ksh \ @@ -18,5 +19,7 @@ dist_pkgdata_SCRIPTS = \ userquota_010_pos.ksh \ userquota_011_pos.ksh \ userquota_012_neg.ksh \ + userquota_013_pos.ksh \ userspace_001_pos.ksh \ - userspace_002_pos.ksh + userspace_002_pos.ksh \ + userspace_003_pos.ksh diff --git a/tests/zfs-tests/tests/functional/userquota/groupspace_003_pos.ksh b/tests/zfs-tests/tests/functional/userquota/groupspace_003_pos.ksh new file mode 100644 index 000000000..7ea8cd563 --- /dev/null +++ b/tests/zfs-tests/tests/functional/userquota/groupspace_003_pos.ksh @@ -0,0 +1,103 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +# +# Copyright (c) 2013 by Delphix. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/userquota/userquota_common.kshlib + +# +# DESCRIPTION: +# Check the user used and groupspace object counts in zfs groupspace +# +# +# STRATEGY: +# 1. set zfs groupquota to a fs +# 2. create objects for different users in the same group +# 3. use zfs groupspace to check the object count +# + +function cleanup +{ + if datasetexists $snapfs; then + log_must $ZFS destroy $snapfs + fi + + log_must $RM -f ${QFILE}_* + log_must cleanup_quota +} + +function group_object_count +{ + typeset fs=$1 + typeset user=$2 + typeset cnt=$($ZFS groupspace -oname,objused $fs | $GREP $user | + $AWK '{print $2}') + echo $cnt +} + +log_onexit cleanup + +log_assert "Check the zfs groupspace object used" + +mkmount_writable $QFS +log_must $ZFS set xattr=sa $QFS + +((user1_cnt = RANDOM % 100 + 1)) +((user2_cnt = RANDOM % 100 + 1)) +log_must user_run $QUSER1 $MKFILES ${QFILE}_1 $user1_cnt +log_must user_run $QUSER2 $MKFILES ${QFILE}_2 $user2_cnt +((grp_cnt = user1_cnt + user2_cnt)) +sync_pool + +typeset snapfs=$QFS@snap + +log_must $ZFS snapshot $snapfs + +log_must eval "$ZFS groupspace $QFS >/dev/null 2>&1" +log_must eval "$ZFS groupspace $snapfs >/dev/null 2>&1" + +for fs in "$QFS" "$snapfs"; do + log_note "check the object count in zfs groupspace $fs" + [[ $(group_object_count $fs $QGROUP) -eq $grp_cnt ]] || + log_fail "expected $grp_cnt" +done + +log_note "file removal" +log_must $RM ${QFILE}_* +sync_pool + +[[ $(group_object_count $QFS $QGROUP) -eq 0 ]] || + log_fail "expected 0 files for $QGROUP" + +[[ $(group_object_count $snapfs $QGROUP) -eq $grp_cnt ]] || + log_fail "expected $grp_cnt files for $QGROUP" + +cleanup +log_pass "Check the zfs groupspace object used pass as expect" diff --git a/tests/zfs-tests/tests/functional/userquota/userquota_001_pos.ksh b/tests/zfs-tests/tests/functional/userquota/userquota_001_pos.ksh index b134a7677..7a4f8f3ba 100755 --- a/tests/zfs-tests/tests/functional/userquota/userquota_001_pos.ksh +++ b/tests/zfs-tests/tests/functional/userquota/userquota_001_pos.ksh @@ -58,7 +58,7 @@ mkmount_writable $QFS log_note "Check the userquota@$QUSER1" log_must $ZFS set userquota@$QUSER1=$UQUOTA_SIZE $QFS log_must user_run $QUSER1 $MKFILE $UQUOTA_SIZE $QFILE -$SYNC +sync_pool log_mustnot user_run $QUSER1 $MKFILE 1 $OFILE cleanup_quota @@ -66,7 +66,7 @@ log_note "Check the groupquota@$QGROUP" log_must $ZFS set groupquota@$QGROUP=$GQUOTA_SIZE $QFS mkmount_writable $QFS log_must user_run $QUSER1 $MKFILE $GQUOTA_SIZE $QFILE -$SYNC +sync_pool log_mustnot user_run $QUSER1 $MKFILE 1 $OFILE cleanup_quota diff --git a/tests/zfs-tests/tests/functional/userquota/userquota_004_pos.ksh b/tests/zfs-tests/tests/functional/userquota/userquota_004_pos.ksh index 6bdcf1c76..749063261 100755 --- a/tests/zfs-tests/tests/functional/userquota/userquota_004_pos.ksh +++ b/tests/zfs-tests/tests/functional/userquota/userquota_004_pos.ksh @@ -50,6 +50,7 @@ log_onexit cleanup log_assert "Check the basic function of {user|group} used" +sync_pool typeset user_used=$(get_value "userused@$QUSER1" $QFS) typeset group_used=$(get_value "groupused@$QGROUP" $QFS) @@ -62,7 +63,7 @@ fi mkmount_writable $QFS log_must user_run $QUSER1 $MKFILE 100m $QFILE -$SYNC +sync_pool user_used=$(get_value "userused@$QUSER1" $QFS) group_used=$(get_value "groupused@$QGROUP" $QFS) diff --git a/tests/zfs-tests/tests/functional/userquota/userquota_010_pos.ksh b/tests/zfs-tests/tests/functional/userquota/userquota_010_pos.ksh index b9260585f..7f7f9672d 100755 --- a/tests/zfs-tests/tests/functional/userquota/userquota_010_pos.ksh +++ b/tests/zfs-tests/tests/functional/userquota/userquota_010_pos.ksh @@ -57,7 +57,7 @@ log_must $ZFS set groupquota@$QGROUP=$GQUOTA_SIZE $QFS mkmount_writable $QFS log_must user_run $QUSER1 $MKFILE $UQUOTA_SIZE $QFILE -$SYNC +sync_pool log_must eval "$ZFS get -p userused@$QUSER1 $QFS >/dev/null 2>&1" log_must eval "$ZFS get -p groupused@$GROUPUSED $QFS >/dev/null 2>&1" diff --git a/tests/zfs-tests/tests/functional/userquota/userquota_013_pos.ksh b/tests/zfs-tests/tests/functional/userquota/userquota_013_pos.ksh new file mode 100644 index 000000000..a84a45524 --- /dev/null +++ b/tests/zfs-tests/tests/functional/userquota/userquota_013_pos.ksh @@ -0,0 +1,77 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +# +# Copyright (c) 2016 by Jinshan Xiong. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/userquota/userquota_common.kshlib + +# +# +# DESCRIPTION: +# Check the basic function of the userobjquota and groupobjquota +# +# +# STRATEGY: +# 1. Set userobjquota and overwrite the quota size +# 2. Creating new object should fail with Disc quota exceeded +# 3. Set groupobjquota and overwrite the quota size +# 4. Creating new object should fail with Disc quota exceeded +# +# + +function cleanup +{ + log_must $RM -f ${QFILE}_* + cleanup_quota +} + +log_onexit cleanup + +log_assert "If creating object exceeds {user|group}objquota count, it will fail" + +mkmount_writable $QFS +log_must $ZFS set xattr=sa $QFS + +log_note "Check the userobjquota@$QUSER1" +log_must $ZFS set userobjquota@$QUSER1=100 $QFS +log_must user_run $QUSER1 $MKFILES ${QFILE}_1 100 +sync_pool +log_mustnot user_run $QUSER1 $MKFILE 1 $OFILE +cleanup_quota + +log_note "Check the groupobjquota@$QGROUP" +log_must $ZFS set groupobjquota@$QGROUP=200 $QFS +mkmount_writable $QFS +log_must user_run $QUSER1 $MKFILES ${QFILE}_2 100 +sync_pool +log_mustnot user_run $QUSER2 $MKFILE 1 $OFILE + +cleanup +log_pass "Creating objects exceeds {user|group}objquota count, it as expect" diff --git a/tests/zfs-tests/tests/functional/userquota/userquota_common.kshlib b/tests/zfs-tests/tests/functional/userquota/userquota_common.kshlib index 771919602..2b50e293e 100644 --- a/tests/zfs-tests/tests/functional/userquota/userquota_common.kshlib +++ b/tests/zfs-tests/tests/functional/userquota/userquota_common.kshlib @@ -38,8 +38,11 @@ function cleanup_quota { if datasetexists $QFS; then log_must $ZFS set userquota@$QUSER1=none $QFS + log_must $ZFS set userobjquota@$QUSER1=none $QFS log_must $ZFS set userquota@$QUSER2=none $QFS + log_must $ZFS set userobjquota@$QUSER2=none $QFS log_must $ZFS set groupquota@$QGROUP=none $QFS + log_must $ZFS set groupobjquota@$QGROUP=none $QFS recovery_writable $QFS fi @@ -47,7 +50,7 @@ function cleanup_quota [[ -f $OFILE ]] && log_must $RM -f $OFILE $SYNC - return 0 + return 0 } # diff --git a/tests/zfs-tests/tests/functional/userquota/userspace_002_pos.ksh b/tests/zfs-tests/tests/functional/userquota/userspace_002_pos.ksh index b29052db6..cb84cf927 100755 --- a/tests/zfs-tests/tests/functional/userquota/userspace_002_pos.ksh +++ b/tests/zfs-tests/tests/functional/userquota/userspace_002_pos.ksh @@ -75,7 +75,7 @@ for fs in "$QFS" "$snapfs"; do log_must eval "$ZFS userspace $fs | $GREP $QUSER1 | $GREP 100M" log_note "check the user used size in zfs userspace $fs" - log_must eval "$ZFS userspace $fs | $GREP $QUSER1 | $GREP 50.0M" + log_must eval "$ZFS userspace $fs | $GREP $QUSER1 | $GREP 50\\.\*M" done log_pass "Check the zfs userspace used and quota" diff --git a/tests/zfs-tests/tests/functional/userquota/userspace_003_pos.ksh b/tests/zfs-tests/tests/functional/userquota/userspace_003_pos.ksh new file mode 100644 index 000000000..421de6581 --- /dev/null +++ b/tests/zfs-tests/tests/functional/userquota/userspace_003_pos.ksh @@ -0,0 +1,116 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +# +# Copyright (c) 2016 by Jinshan Xiong. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/userquota/userquota_common.kshlib + +# +# DESCRIPTION: +# Check the user used object accounting in zfs userspace +# +# +# STRATEGY: +# 1. create a bunch of files by specific users +# 2. use zfs userspace to check the used objects +# 3. change the owner of test files and verify object count +# 4. delete files and verify object count +# + +function cleanup +{ + if datasetexists $snapfs; then + log_must $ZFS destroy $snapfs + fi + + log_must $RM -f ${QFILE}_* + log_must cleanup_quota +} + +function user_object_count +{ + typeset fs=$1 + typeset user=$2 + typeset cnt=$($ZFS userspace -oname,objused $fs | + $AWK /$user/'{print $2}') + echo $cnt +} + +log_onexit cleanup + +log_assert "Check the zfs userspace object used" + +mkmount_writable $QFS +log_must $ZFS set xattr=sa $QFS + +((user1_cnt = RANDOM % 100 + 1)) +((user2_cnt = RANDOM % 100 + 1)) + +log_must user_run $QUSER1 $MKFILES ${QFILE}_1 $user1_cnt +log_must user_run $QUSER2 $MKFILES ${QFILE}_2 $user2_cnt +sync_pool + +typeset snapfs=$QFS@snap + +log_must $ZFS snapshot $snapfs + +log_must eval "$ZFS userspace $QFS >/dev/null 2>&1" +log_must eval "$ZFS userspace $snapfs >/dev/null 2>&1" + +for fs in "$QFS" "$snapfs"; do + log_note "check the user used objects in zfs userspace $fs" + [[ $(user_object_count $fs $QUSER1) -eq $user1_cnt ]] || + log_fail "expected $user1_cnt" + [[ $(user_object_count $fs $QUSER2) -eq $user2_cnt ]] || + log_fail "expected $user2_cnt" +done + +log_note "change the owner of files" +log_must $CHOWN $QUSER2 ${QFILE}_1* +sync_pool + +[[ $(user_object_count $QFS $QUSER1) -eq 0 ]] || + log_fail "expected 0 files for $QUSER1" + +[[ $(user_object_count $snapfs $QUSER1) -eq $user1_cnt ]] || + log_fail "expected $user_cnt files for $QUSER1 in snapfs" + +[[ $(user_object_count $QFS $QUSER2) -eq $((user1_cnt+user2_cnt)) ]] || + log_fail "expected $((user1_cnt+user2_cnt)) files for $QUSER2" + +log_note "file removal" +log_must $RM ${QFILE}_* +sync_pool + +[[ $(user_object_count $QFS $QUSER2) -eq 0 ]] || + log_fail "expected 0 files for $QUSER2" + +cleanup +log_pass "Check the zfs userspace object used" diff --git a/zfs-script-config.sh.in b/zfs-script-config.sh.in index 594958fa0..d1ba71d76 100644 --- a/zfs-script-config.sh.in +++ b/zfs-script-config.sh.in @@ -29,6 +29,9 @@ export ZPOOL=${CMDDIR}/zpool/zpool export ZTEST=${CMDDIR}/ztest/ztest export ZPIOS=${CMDDIR}/zpios/zpios export RAIDZ_TEST=${CMDDIR}/raidz_test/raidz_test +export ARC_SUMMARY=${CMDDIR}/arc_summary/arc_summary.py +export ARCSTAT=${CMDDIR}/arcstat/arcstat.py +export DBUFSTAT=${CMDDIR}/dbufstat/dbufstat.py export COMMON_SH=${SCRIPTDIR}/common.sh export ZFS_SH=${SCRIPTDIR}/zfs.sh |