Add support for user/group dnode accounting & quota

This patch tracks dnode usage for each user/group in the
DMU_USER/GROUPUSED_OBJECT ZAPs. ZAP entries dedicated to dnode
accounting have the key prefixed with "obj-" followed by the UID/GID
in string format (as done for the block accounting).
A new SPA feature has been added for dnode accounting as well as
a new ZPL version. The SPA feature must be enabled in the pool
before upgrading the zfs filesystem. During the zfs version upgrade,
a "quotacheck" will be executed by marking all dnode as dirty.

ZoL-bug-id: https://github.com/zfsonlinux/zfs/issues/3500

Signed-off-by: Jinshan Xiong <[email protected]>
Signed-off-by: Johann Lombardi <[email protected]>

43 files changed, 1119 insertions, 98 deletions

diff --git a/cmd/zdb/zdb.c b/cmd/zdb/zdb.c
index 611e92294..7a0f55f75 100644
--- a/cmd/zdb/zdb.c
+++ b/cmd/zdb/zdb.c
@@ -1946,11 +1946,13 @@ dump_object(objset_t *os, uint64_t object, int verbosity, int *print_header)
 	}
 
 	if (verbosity >= 4) {
-		(void) printf("\tdnode flags: %s%s%s\n",
+		(void) printf("\tdnode flags: %s%s%s%s\n",
 		    (dn->dn_phys->dn_flags & DNODE_FLAG_USED_BYTES) ?
 		    "USED_BYTES " : "",
 		    (dn->dn_phys->dn_flags & DNODE_FLAG_USERUSED_ACCOUNTED) ?
 		    "USERUSED_ACCOUNTED " : "",
+		    (dn->dn_phys->dn_flags & DNODE_FLAG_USEROBJUSED_ACCOUNTED) ?
+		    "USEROBJUSED_ACCOUNTED " : "",
 		    (dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR) ?
 		    "SPILL_BLKPTR" : "");
 		(void) printf("\tdnode maxblkid: %llu\n",
diff --git a/cmd/zfs/zfs_main.c b/cmd/zfs/zfs_main.c
index fedc1a04d..8a49e31e5 100644
--- a/cmd/zfs/zfs_main.c
+++ b/cmd/zfs/zfs_main.c
@@ -2223,10 +2223,14 @@ enum us_field_types {
 	USFIELD_TYPE,
 	USFIELD_NAME,
 	USFIELD_USED,
-	USFIELD_QUOTA
+	USFIELD_QUOTA,
+	USFIELD_OBJUSED,
+	USFIELD_OBJQUOTA
 };
-static char *us_field_hdr[] = { "TYPE", "NAME", "USED", "QUOTA" };
-static char *us_field_names[] = { "type", "name", "used", "quota" };
+static char *us_field_hdr[] = { "TYPE", "NAME", "USED", "QUOTA",
+				    "OBJUSED", "OBJQUOTA" };
+static char *us_field_names[] = { "type", "name", "used", "quota",
+				    "objused", "objquota" };
 #define	USFIELD_LAST	(sizeof (us_field_names) / sizeof (char *))
 
 #define	USTYPE_PSX_GRP	(1 << 0)
@@ -2374,6 +2378,20 @@ compare_nums:
 	return (0);
 }
 
+static boolean_t
+zfs_prop_is_user(unsigned p)
+{
+	return (p == ZFS_PROP_USERUSED || p == ZFS_PROP_USERQUOTA ||
+	    p == ZFS_PROP_USEROBJUSED || p == ZFS_PROP_USEROBJQUOTA);
+}
+
+static boolean_t
+zfs_prop_is_group(unsigned p)
+{
+	return (p == ZFS_PROP_GROUPUSED || p == ZFS_PROP_GROUPQUOTA ||
+	    p == ZFS_PROP_GROUPOBJUSED || p == ZFS_PROP_GROUPOBJQUOTA);
+}
+
 static inline const char *
 us_type2str(unsigned field_type)
 {
@@ -2463,7 +2481,7 @@ userspace_cb(void *arg, const char *domain, uid_t rid, uint64_t space)
 
 	if (cb->cb_sid2posix || domain == NULL || domain[0] == '\0') {
 		/* POSIX or -i */
-		if (prop == ZFS_PROP_GROUPUSED || prop == ZFS_PROP_GROUPQUOTA) {
+		if (zfs_prop_is_group(prop)) {
 			type = USTYPE_PSX_GRP;
 			if (!cb->cb_numname) {
 				struct group *g;
@@ -2538,10 +2556,22 @@ userspace_cb(void *arg, const char *domain, uid_t rid, uint64_t space)
 		propname = "used";
 		if (!nvlist_exists(props, "quota"))
 			(void) nvlist_add_uint64(props, "quota", 0);
-	} else {
+	} else if (prop == ZFS_PROP_USERQUOTA || prop == ZFS_PROP_GROUPQUOTA) {
 		propname = "quota";
 		if (!nvlist_exists(props, "used"))
 			(void) nvlist_add_uint64(props, "used", 0);
+	} else if (prop == ZFS_PROP_USEROBJUSED ||
+		    prop == ZFS_PROP_GROUPOBJUSED) {
+		propname = "objused";
+		if (!nvlist_exists(props, "objquota"))
+			(void) nvlist_add_uint64(props, "objquota", 0);
+	} else if (prop == ZFS_PROP_USEROBJQUOTA ||
+		    prop == ZFS_PROP_GROUPOBJQUOTA) {
+		propname = "objquota";
+		if (!nvlist_exists(props, "objused"))
+			(void) nvlist_add_uint64(props, "objused", 0);
+	} else {
+		return (-1);
 	}
 	sizeidx = us_field_index(propname);
 	if (sizelen > cb->cb_width[sizeidx])
@@ -2574,7 +2604,7 @@ print_us_node(boolean_t scripted, boolean_t parsable, int *fields, int types,
 		data_type_t type;
 		uint32_t val32;
 		uint64_t val64;
-		char *strval = NULL;
+		char *strval = "-";
 
 		while ((nvp = nvlist_next_nvpair(nvl, nvp)) != NULL) {
 			if (strcmp(nvpair_name(nvp),
@@ -2582,7 +2612,7 @@ print_us_node(boolean_t scripted, boolean_t parsable, int *fields, int types,
 				break;
 		}
 
-		type = nvpair_type(nvp);
+		type = nvp == NULL ? DATA_TYPE_UNKNOWN : nvpair_type(nvp);
 		switch (type) {
 		case DATA_TYPE_UINT32:
 			(void) nvpair_value_uint32(nvp, &val32);
@@ -2593,13 +2623,16 @@ print_us_node(boolean_t scripted, boolean_t parsable, int *fields, int types,
 		case DATA_TYPE_STRING:
 			(void) nvpair_value_string(nvp, &strval);
 			break;
+		case DATA_TYPE_UNKNOWN:
+			break;
 		default:
 			(void) fprintf(stderr, "invalid data type\n");
 		}
 
 		switch (field) {
 		case USFIELD_TYPE:
-			strval = (char *)us_type2str(val32);
+			if (type == DATA_TYPE_UINT32)
+				strval = (char *)us_type2str(val32);
 			break;
 		case USFIELD_NAME:
 			if (type == DATA_TYPE_UINT64) {
@@ -2610,6 +2643,8 @@ print_us_node(boolean_t scripted, boolean_t parsable, int *fields, int types,
 			break;
 		case USFIELD_USED:
 		case USFIELD_QUOTA:
+		case USFIELD_OBJUSED:
+		case USFIELD_OBJQUOTA:
 			if (type == DATA_TYPE_UINT64) {
 				if (parsable) {
 					(void) sprintf(valstr, "%llu",
@@ -2618,7 +2653,8 @@ print_us_node(boolean_t scripted, boolean_t parsable, int *fields, int types,
 					zfs_nicenum(val64, valstr,
 					    sizeof (valstr));
 				}
-				if (field == USFIELD_QUOTA &&
+				if ((field == USFIELD_QUOTA ||
+				    field == USFIELD_OBJQUOTA) &&
 				    strcmp(valstr, "0") == 0)
 					strval = "none";
 				else
@@ -2690,7 +2726,7 @@ zfs_do_userspace(int argc, char **argv)
 	uu_avl_t *avl_tree;
 	uu_avl_walk_t *walk;
 	char *delim;
-	char deffields[] = "type,name,used,quota";
+	char deffields[] = "type,name,used,quota,objused,objquota";
 	char *ofield = NULL;
 	char *tfield = NULL;
 	int cfield = 0;
@@ -2839,11 +2875,12 @@ zfs_do_userspace(int argc, char **argv)
 		cb.cb_width[i] = strlen(gettext(us_field_hdr[i]));
 
 	for (p = 0; p < ZFS_NUM_USERQUOTA_PROPS; p++) {
-		if (((p == ZFS_PROP_USERUSED || p == ZFS_PROP_USERQUOTA) &&
+		if ((zfs_prop_is_user(p) &&
 		    !(types & (USTYPE_PSX_USR | USTYPE_SMB_USR))) ||
-		    ((p == ZFS_PROP_GROUPUSED || p == ZFS_PROP_GROUPQUOTA) &&
+		    (zfs_prop_is_group(p) &&
 		    !(types & (USTYPE_PSX_GRP | USTYPE_SMB_GRP))))
 			continue;
+
 		cb.cb_prop = p;
 		if ((ret = zfs_userspace(zhp, p, userspace_cb, &cb)) != 0)
 			return (ret);
@@ -4099,6 +4136,11 @@ zfs_do_receive(int argc, char **argv)
 #define	ZFS_DELEG_PERM_GROUPQUOTA	"groupquota"
 #define	ZFS_DELEG_PERM_USERUSED		"userused"
 #define	ZFS_DELEG_PERM_GROUPUSED	"groupused"
+#define	ZFS_DELEG_PERM_USEROBJQUOTA	"userobjquota"
+#define	ZFS_DELEG_PERM_GROUPOBJQUOTA	"groupobjquota"
+#define	ZFS_DELEG_PERM_USEROBJUSED	"userobjused"
+#define	ZFS_DELEG_PERM_GROUPOBJUSED	"groupobjused"
+
 #define	ZFS_DELEG_PERM_HOLD		"hold"
 #define	ZFS_DELEG_PERM_RELEASE		"release"
 #define	ZFS_DELEG_PERM_DIFF		"diff"
@@ -4129,6 +4171,10 @@ static zfs_deleg_perm_tab_t zfs_deleg_perm_tbl[] = {
 	{ ZFS_DELEG_PERM_USERPROP, ZFS_DELEG_NOTE_USERPROP },
 	{ ZFS_DELEG_PERM_USERQUOTA, ZFS_DELEG_NOTE_USERQUOTA },
 	{ ZFS_DELEG_PERM_USERUSED, ZFS_DELEG_NOTE_USERUSED },
+	{ ZFS_DELEG_PERM_USEROBJQUOTA, ZFS_DELEG_NOTE_USEROBJQUOTA },
+	{ ZFS_DELEG_PERM_USEROBJUSED, ZFS_DELEG_NOTE_USEROBJUSED },
+	{ ZFS_DELEG_PERM_GROUPOBJQUOTA, ZFS_DELEG_NOTE_GROUPOBJQUOTA },
+	{ ZFS_DELEG_PERM_GROUPOBJUSED, ZFS_DELEG_NOTE_GROUPOBJUSED },
 	{ NULL, ZFS_DELEG_NOTE_NONE }
 };
 
@@ -4206,6 +4252,10 @@ deleg_perm_type(zfs_deleg_note_t note)
 	case ZFS_DELEG_NOTE_USERPROP:
 	case ZFS_DELEG_NOTE_USERQUOTA:
 	case ZFS_DELEG_NOTE_USERUSED:
+	case ZFS_DELEG_NOTE_USEROBJQUOTA:
+	case ZFS_DELEG_NOTE_USEROBJUSED:
+	case ZFS_DELEG_NOTE_GROUPOBJQUOTA:
+	case ZFS_DELEG_NOTE_GROUPOBJUSED:
 		/* other */
 		return (gettext("other"));
 	default:
@@ -4709,6 +4759,19 @@ deleg_perm_comment(zfs_deleg_note_t note)
 	case ZFS_DELEG_NOTE_USERUSED:
 		str = gettext("Allows reading any userused@... property");
 		break;
+	case ZFS_DELEG_NOTE_USEROBJQUOTA:
+		str = gettext("Allows accessing any userobjquota@... property");
+		break;
+	case ZFS_DELEG_NOTE_GROUPOBJQUOTA:
+		str = gettext("Allows accessing any \n\t\t\t\t"
+		    "groupobjquota@... property");
+		break;
+	case ZFS_DELEG_NOTE_GROUPOBJUSED:
+		str = gettext("Allows reading any groupobjused@... property");
+		break;
+	case ZFS_DELEG_NOTE_USEROBJUSED:
+		str = gettext("Allows reading any userobjused@... property");
+		break;
 		/* other */
 	default:
 		str = "";
diff --git a/configure.ac b/configure.ac
index edcf29958..f01a4d872 100644
--- a/configure.ac
+++ b/configure.ac
@@ -274,6 +274,7 @@ AC_CONFIG_FILES([
 	tests/zfs-tests/tests/functional/threadsappend/Makefile
 	tests/zfs-tests/tests/functional/truncate/Makefile
 	tests/zfs-tests/tests/functional/userquota/Makefile
+	tests/zfs-tests/tests/functional/upgrade/Makefile
 	tests/zfs-tests/tests/functional/vdev_zaps/Makefile
 	tests/zfs-tests/tests/functional/write_dirs/Makefile
 	tests/zfs-tests/tests/functional/xattr/Makefile
diff --git a/include/sys/dmu.h b/include/sys/dmu.h
index b67acb52c..ec89e9018 100644
--- a/include/sys/dmu.h
+++ b/include/sys/dmu.h
@@ -257,6 +257,12 @@ void zfs_znode_byteswap(void *buf, size_t size);
 #define	DMU_GROUPUSED_OBJECT	(-2ULL)
 
 /*
+ * Zap prefix for object accounting in DMU_{USER,GROUP}USED_OBJECT.
+ */
+#define	DMU_OBJACCT_PREFIX	"obj-"
+#define	DMU_OBJACCT_PREFIX_LEN	4
+
+/*
  * artificial blkids for bonus buffer and spill blocks
  */
 #define	DMU_BONUS_BLKID		(-1ULL)
diff --git a/include/sys/dmu_objset.h b/include/sys/dmu_objset.h
index 1674897c2..68fb5cffb 100644
--- a/include/sys/dmu_objset.h
+++ b/include/sys/dmu_objset.h
@@ -56,6 +56,7 @@ struct dmu_tx;
 	(arc_buf_size(buf) > OBJSET_OLD_PHYS_SIZE)
 
 #define	OBJSET_FLAG_USERACCOUNTING_COMPLETE	(1ULL<<0)
+#define	OBJSET_FLAG_USEROBJACCOUNTING_COMPLETE	(1ULL<<1)
 
 typedef struct objset_phys {
 	dnode_phys_t os_meta_dnode;
@@ -68,6 +69,8 @@ typedef struct objset_phys {
 	dnode_phys_t os_groupused_dnode;
 } objset_phys_t;
 
+typedef int (*dmu_objset_upgrade_cb_t)(objset_t *);
+
 struct objset {
 	/* Immutable: */
 	struct dsl_dataset *os_dsl_dataset;
@@ -125,6 +128,13 @@ struct objset {
 	kmutex_t os_user_ptr_lock;
 	void *os_user_ptr;
 	sa_os_t *os_sa;
+
+	/* kernel thread to upgrade this dataset */
+	kmutex_t os_upgrade_lock;
+	taskqid_t os_upgrade_id;
+	dmu_objset_upgrade_cb_t os_upgrade_cb;
+	boolean_t os_upgrade_exit;
+	int os_upgrade_status;
 };
 
 #define	DMU_META_OBJSET		0
@@ -173,6 +183,17 @@ void dmu_objset_userquota_get_ids(dnode_t *dn, boolean_t before, dmu_tx_t *tx);
 boolean_t dmu_objset_userused_enabled(objset_t *os);
 int dmu_objset_userspace_upgrade(objset_t *os);
 boolean_t dmu_objset_userspace_present(objset_t *os);
+boolean_t dmu_objset_userobjused_enabled(objset_t *os);
+void dmu_objset_userobjspace_upgrade(objset_t *os);
+boolean_t dmu_objset_userobjspace_present(objset_t *os);
+
+static inline boolean_t dmu_objset_userobjspace_upgradable(objset_t *os)
+{
+	return (dmu_objset_type(os) == DMU_OST_ZFS &&
+	    dmu_objset_userobjused_enabled(os) &&
+	    !dmu_objset_userobjspace_present(os));
+}
+
 int dmu_fsname(const char *snapname, char *buf);
 
 void dmu_objset_evict_done(objset_t *os);
diff --git a/include/sys/dnode.h b/include/sys/dnode.h
index fe36e5989..fe4cd3e4b 100644
--- a/include/sys/dnode.h
+++ b/include/sys/dnode.h
@@ -126,11 +126,14 @@ enum dnode_dirtycontext {
 };
 
 /* Is dn_used in bytes?  if not, it's in multiples of SPA_MINBLOCKSIZE */
-#define	DNODE_FLAG_USED_BYTES		(1<<0)
-#define	DNODE_FLAG_USERUSED_ACCOUNTED	(1<<1)
+#define	DNODE_FLAG_USED_BYTES			(1 << 0)
+#define	DNODE_FLAG_USERUSED_ACCOUNTED		(1 << 1)
 
 /* Does dnode have a SA spill blkptr in bonus? */
-#define	DNODE_FLAG_SPILL_BLKPTR	(1<<2)
+#define	DNODE_FLAG_SPILL_BLKPTR			(1 << 2)
+
+/* User/Group dnode accounting */
+#define	DNODE_FLAG_USEROBJUSED_ACCOUNTED	(1 << 3)
 
 typedef struct dnode_phys {
 	uint8_t dn_type;		/* dmu_object_type_t */
diff --git a/include/sys/dsl_deleg.h b/include/sys/dsl_deleg.h
index 59e8e0555..d399d1da9 100644
--- a/include/sys/dsl_deleg.h
+++ b/include/sys/dsl_deleg.h
@@ -51,8 +51,12 @@ extern "C" {
 #define	ZFS_DELEG_PERM_VSCAN		"vscan"
 #define	ZFS_DELEG_PERM_USERQUOTA	"userquota"
 #define	ZFS_DELEG_PERM_GROUPQUOTA	"groupquota"
+#define	ZFS_DELEG_PERM_USEROBJQUOTA	"userobjquota"
+#define	ZFS_DELEG_PERM_GROUPOBJQUOTA	"groupobjquota"
 #define	ZFS_DELEG_PERM_USERUSED		"userused"
 #define	ZFS_DELEG_PERM_GROUPUSED	"groupused"
+#define	ZFS_DELEG_PERM_USEROBJUSED	"userobjused"
+#define	ZFS_DELEG_PERM_GROUPOBJUSED	"groupobjused"
 #define	ZFS_DELEG_PERM_HOLD		"hold"
 #define	ZFS_DELEG_PERM_RELEASE		"release"
 #define	ZFS_DELEG_PERM_DIFF		"diff"
diff --git a/include/sys/fs/zfs.h b/include/sys/fs/zfs.h
index 9d3008934..5c93f53de 100644
--- a/include/sys/fs/zfs.h
+++ b/include/sys/fs/zfs.h
@@ -171,6 +171,10 @@ typedef enum {
 	ZFS_PROP_USERQUOTA,
 	ZFS_PROP_GROUPUSED,
 	ZFS_PROP_GROUPQUOTA,
+	ZFS_PROP_USEROBJUSED,
+	ZFS_PROP_USEROBJQUOTA,
+	ZFS_PROP_GROUPOBJUSED,
+	ZFS_PROP_GROUPOBJQUOTA,
 	ZFS_NUM_USERQUOTA_PROPS
 } zfs_userquota_prop_t;
 
diff --git a/include/sys/spa_impl.h b/include/sys/spa_impl.h
index 7b9e1ee0c..cb1d16ad5 100644
--- a/include/sys/spa_impl.h
+++ b/include/sys/spa_impl.h
@@ -277,6 +277,8 @@ struct spa {
 	 */
 	spa_config_lock_t spa_config_lock[SCL_LOCKS]; /* config changes */
 	refcount_t	spa_refcount;		/* number of opens */
+
+	taskq_t		*spa_upgrade_taskq;	/* taskq for upgrade jobs */
 };
 
 extern char *spa_config_path;
diff --git a/include/sys/zfs_vfsops.h b/include/sys/zfs_vfsops.h
index efaefdacc..b59ace5b1 100644
--- a/include/sys/zfs_vfsops.h
+++ b/include/sys/zfs_vfsops.h
@@ -110,6 +110,8 @@ typedef struct zfs_sb {
 	kmutex_t	z_lock;
 	uint64_t	z_userquota_obj;
 	uint64_t	z_groupquota_obj;
+	uint64_t	z_userobjquota_obj;
+	uint64_t	z_groupobjquota_obj;
 	uint64_t	z_replay_eof;	/* New end of file - replay only */
 	sa_attr_type_t	*z_attr_table;	/* SA attr mapping->id */
 	uint64_t	z_hold_size;	/* znode hold array size */
@@ -190,6 +192,8 @@ extern boolean_t zfs_owner_overquota(zfs_sb_t *zsb, struct znode *,
     boolean_t isgroup);
 extern boolean_t zfs_fuid_overquota(zfs_sb_t *zsb, boolean_t isgroup,
     uint64_t fuid);
+extern boolean_t zfs_fuid_overobjquota(zfs_sb_t *zsb, boolean_t isgroup,
+    uint64_t fuid);
 extern int zfs_set_version(zfs_sb_t *zsb, uint64_t newvers);
 extern int zfs_get_zplprop(objset_t *os, zfs_prop_t prop,
     uint64_t *value);
diff --git a/include/zfeature_common.h b/include/zfeature_common.h
index f05480181..acf76381b 100644
--- a/include/zfeature_common.h
+++ b/include/zfeature_common.h
@@ -54,6 +54,7 @@ typedef enum spa_feature {
 	SPA_FEATURE_SHA512,
 	SPA_FEATURE_SKEIN,
 	SPA_FEATURE_EDONR,
+	SPA_FEATURE_USEROBJ_ACCOUNTING,
 	SPA_FEATURES
 } spa_feature_t;
 
diff --git a/include/zfs_deleg.h b/include/zfs_deleg.h
index 16133c59f..95db9921f 100644
--- a/include/zfs_deleg.h
+++ b/include/zfs_deleg.h
@@ -63,6 +63,10 @@ typedef enum {
 	ZFS_DELEG_NOTE_GROUPQUOTA,
 	ZFS_DELEG_NOTE_USERUSED,
 	ZFS_DELEG_NOTE_GROUPUSED,
+	ZFS_DELEG_NOTE_USEROBJQUOTA,
+	ZFS_DELEG_NOTE_GROUPOBJQUOTA,
+	ZFS_DELEG_NOTE_USEROBJUSED,
+	ZFS_DELEG_NOTE_GROUPOBJUSED,
 	ZFS_DELEG_NOTE_HOLD,
 	ZFS_DELEG_NOTE_RELEASE,
 	ZFS_DELEG_NOTE_DIFF,
diff --git a/lib/libzfs/libzfs_dataset.c b/lib/libzfs/libzfs_dataset.c
index 5ecf96985..7d8179d1f 100755
--- a/lib/libzfs/libzfs_dataset.c
+++ b/lib/libzfs/libzfs_dataset.c
@@ -944,7 +944,9 @@ zfs_valid_proplist(libzfs_handle_t *hdl, zfs_type_t type, nvlist_t *nvl,
 			}
 
 			if (uqtype != ZFS_PROP_USERQUOTA &&
-			    uqtype != ZFS_PROP_GROUPQUOTA) {
+			    uqtype != ZFS_PROP_GROUPQUOTA &&
+			    uqtype != ZFS_PROP_USEROBJQUOTA &&
+			    uqtype != ZFS_PROP_GROUPOBJQUOTA) {
 				zfs_error_aux(hdl,
 				    dgettext(TEXT_DOMAIN, "'%s' is readonly"),
 				    propname);
@@ -2741,8 +2743,12 @@ userquota_propname_decode(const char *propname, boolean_t zoned,
 		return (EINVAL);
 	*typep = type;
 
-	isuser = (type == ZFS_PROP_USERQUOTA || type == ZFS_PROP_USERUSED);
-	isgroup = (type == ZFS_PROP_GROUPQUOTA || type == ZFS_PROP_GROUPUSED);
+	isuser = (type == ZFS_PROP_USERQUOTA || type == ZFS_PROP_USERUSED ||
+		    type == ZFS_PROP_USEROBJQUOTA ||
+		    type == ZFS_PROP_USEROBJUSED);
+	isgroup = (type == ZFS_PROP_GROUPQUOTA || type == ZFS_PROP_GROUPUSED ||
+		    type == ZFS_PROP_GROUPOBJQUOTA ||
+		    type == ZFS_PROP_GROUPOBJUSED);
 
 	cp = strchr(propname, '@') + 1;
 
@@ -2875,7 +2881,8 @@ zfs_prop_get_userquota(zfs_handle_t *zhp, const char *propname,
 		(void) snprintf(propbuf, proplen, "%llu",
 		    (u_longlong_t)propvalue);
 	} else if (propvalue == 0 &&
-	    (type == ZFS_PROP_USERQUOTA || type == ZFS_PROP_GROUPQUOTA)) {
+	    (type == ZFS_PROP_USERQUOTA || type == ZFS_PROP_GROUPQUOTA ||
+	    type == ZFS_PROP_USEROBJQUOTA || type == ZFS_PROP_GROUPOBJQUOTA)) {
 		(void) strlcpy(propbuf, "none", proplen);
 	} else {
 		zfs_nicenum(propvalue, propbuf, proplen);
@@ -4333,6 +4340,13 @@ zfs_userspace(zfs_handle_t *zhp, zfs_userquota_prop_t type,
 		if (zfs_ioctl(hdl, ZFS_IOC_USERSPACE_MANY, &zc) != 0) {
 			char errbuf[1024];
 
+			if ((errno == ENOTSUP &&
+			    (type == ZFS_PROP_USEROBJUSED ||
+			    type == ZFS_PROP_GROUPOBJUSED ||
+			    type == ZFS_PROP_USEROBJQUOTA ||
+			    type == ZFS_PROP_GROUPOBJQUOTA)))
+				break;
+
 			(void) snprintf(errbuf, sizeof (errbuf),
 			    dgettext(TEXT_DOMAIN,
 			    "cannot get used/quota for %s"), zc.zc_name);
diff --git a/man/man5/zpool-features.5 b/man/man5/zpool-features.5
index ffefd4129..ccc7ab47e 100644
--- a/man/man5/zpool-features.5
+++ b/man/man5/zpool-features.5
@@ -563,5 +563,31 @@ Booting off of pools using \fBedonr\fR is \fBNOT\fR supported
 -- any attempt to enable \fBedonr\fR on a root pool will fail with an
 error.
 
+.sp
+.ne 2
+.na
+\fB\fBuserobj_accounting\fR\fR
+.ad
+.RS 4n
+.TS
+l l .
+GUID	org.zfsonlinux:userobj_accounting
+READ\-ONLY COMPATIBLE	yes
+DEPENDENCIES	extensible_dataset
+.TE
+
+This feature allows administrators to account the object usage information
+by user and group.
+
+This feature becomes \fBactive\fR as soon as it is enabled and will never
+return to being \fBenabled\fR. Each filesystem will be upgraded automatically
+when remounted, or when new files are created under that filesystem.
+The upgrade can also be started manually on filesystems by running
+`zfs set version=current <pool/fs>`. The upgrade process runs in the background
+and may take a while to complete for filesystems containing a large number of
+files.
+
+.RE
+
 .SH "SEE ALSO"
 \fBzpool\fR(8)
diff --git a/man/man8/zfs.8 b/man/man8/zfs.8
index e543ba51d..d8c150629 100644
--- a/man/man8/zfs.8
+++ b/man/man8/zfs.8
@@ -663,6 +663,8 @@ The amount of space consumed by snapshots of this dataset. In particular, it is
 .sp
 .ne 2
 .na
+\fB\fBuserobjused@\fR\fIuser\fR\fR
+.br
 \fB\fBuserused@\fR\fIuser\fR\fR
 .ad
 .sp .6
@@ -699,6 +701,11 @@ The \fBuserused@\fR... properties are not displayed by \fBzfs get all\fR. The us
 .RE
 Files created on Linux always have POSIX owners.
 
+.RS 4n
+The \fBuserobjused\fR is similar to \fBuserused\fR but instead it counts the number of objects consumed by \fIuser\fR. This feature doesn't count the internal objects used by ZFS, therefore it may under count a few objects comparing with the results of third-party tool such as \fBdfs -i\fR.
+When the property \fBxattr=on\fR is set on a fileset, ZFS will create additional objects per-file to store extended attributes. These additional objects are reflected in the \fBuserobjused\fR value and are counted against the user's \fBuserobjquota\fR. When a filesystem is configured to use \fBxattr=sa\fR no additional internal objects are required.
+.RE
+
 .sp
 .ne 2
 .na
@@ -713,6 +720,8 @@ This property is set to the number of user holds on this snapshot. User holds ar
 .ne 2
 .na
 \fB\fBgroupused@\fR\fIgroup\fR\fR
+.br
+\fB\fBgroupobjused@\fR\fIgroup\fR\fR
 .ad
 .sp .6
 .RS 4n
@@ -721,6 +730,11 @@ The amount of space consumed by the specified group in this dataset. Space is ch
 Unprivileged users can only access their own groups' space usage. The root user, or a user who has been granted the \fBgroupused\fR privilege with \fBzfs allow\fR, can access all groups' usage.
 .RE
 
+.RS 4n
+The \fBgroupobjused\fR is similar to \fBgroupused\fR but instead it counts the number of objects consumed by \fIgroup\fR.
+When the property \fBxattr=on\fR is set on a fileset, ZFS will create additional objects per-file to store extended attributes. These additional objects are reflected in the \fBgroupobjused\fR value and are counted against the group's \fBgroupobjquota.\fR. When a filesystem is configured to use \fBxattr=sa\fR no additional internal objects are required.
+.RE
+
 .sp
 .ne 2
 .na
@@ -1081,6 +1095,8 @@ a zone. This feature must be enabled to be used (see \fBzpool-features\fR(5)).
 .ne 2
 .na
 \fB\fBuserquota@\fR\fIuser\fR=\fBnone\fR | \fIsize\fR\fR
+.br
+\fB\fBuserobjquota@\fR\fIuser\fR=\fBnone\fR | \fIcount\fR\fR
 .ad
 .sp .6
 .RS 4n
@@ -1118,16 +1134,26 @@ This property is not available on volumes, on file systems before version 4, or
 .RE
 Files created on Linux always have POSIX owners.
 
+.RS 4
+The \fBuserobjquota\fR is similar to \fBuserquota\fR but it limits the number of objects a \fIuser\fR can create.
+Please refer to \fBuserobjused\fR for more information about how ZFS counts object usage.
+.RE
+
 .sp
 .ne 2
 .na
 \fB\fBgroupquota@\fR\fIgroup\fR=\fBnone\fR\fR | \fIsize\fR
+.br
+\fB\fBgroupobjquota@\fR\fIgroup\fR=\fBnone\fR\fR | \fIcount\fR
 .ad
 .sp .6
 .RS 4n
 Limits the amount of space consumed by the specified group. Group space consumption is identified by the \fBuserquota@\fR\fIuser\fR property.
 .sp
 Unprivileged users can access only their own groups' space usage. The root user, or a user who has been granted the \fBgroupquota\fR privilege with \fBzfs allow\fR, can get and set all groups' quotas.
+
+The \fBgroupobjquota\fR is similar to \fBgroupquota\fR but it limits that the \fIgroup\fR can consume \fIcount\fR number of objects at most.
+Please refer to \fBuserobjused\fR for more information about how zfs counts object usage.
 .RE
 
 .sp
@@ -2386,8 +2412,8 @@ Upgrades to the specified \fIversion\fR. If the \fB-V\fR flag is not specified,
 .sp .6
 .RS 4n
 Displays space consumed by, and quotas on, each user in the specified
-filesystem or snapshot. This corresponds to the \fBuserused@\fR\fIuser\fR and
-\fBuserquota@\fR\fIuser\fR properties.
+filesystem or snapshot. This corresponds to the \fBuserused@\fR\fIuser\fR, \fBuserobjused@\fR\fIuser\fR,
+\fBuserquota@\fR\fIuser\fR, and \fBuserobjquota@\fR\fIuser\fR properties.
 .sp
 .ne 2
 .na
@@ -3141,10 +3167,14 @@ send             subcommand
 share            subcommand     Allows sharing file systems over NFS or SMB
                                 protocols
 snapshot         subcommand     Must also have the 'mount' ability
+groupobjquota    other          Allows accessing any groupobjquota@... property
 groupquota       other          Allows accessing any groupquota@... property
+groupobjused     other          Allows reading any groupobjused@... property
 groupused        other          Allows reading any groupused@... property
 userprop         other          Allows changing any user property
+userobjquota     other          Allows accessing any userobjquota@... property
 userquota        other          Allows accessing any userquota@... property
+userobjused      other          Allows reading any userobjused@... property
 userused         other          Allows reading any userused@... property
 
 acltype          property
diff --git a/module/zcommon/zfs_deleg.c b/module/zcommon/zfs_deleg.c
index f6e41da9d..647a24e5f 100644
--- a/module/zcommon/zfs_deleg.c
+++ b/module/zcommon/zfs_deleg.c
@@ -62,6 +62,10 @@ zfs_deleg_perm_tab_t zfs_deleg_perm_tab[] = {
 	{ZFS_DELEG_PERM_GROUPQUOTA},
 	{ZFS_DELEG_PERM_USERUSED},
 	{ZFS_DELEG_PERM_GROUPUSED},
+	{ZFS_DELEG_PERM_USEROBJQUOTA},
+	{ZFS_DELEG_PERM_GROUPOBJQUOTA},
+	{ZFS_DELEG_PERM_USEROBJUSED},
+	{ZFS_DELEG_PERM_GROUPOBJUSED},
 	{ZFS_DELEG_PERM_HOLD},
 	{ZFS_DELEG_PERM_RELEASE},
 	{NULL}
diff --git a/module/zcommon/zfs_prop.c b/module/zcommon/zfs_prop.c
index 029075ebe..1802750f9 100644
--- a/module/zcommon/zfs_prop.c
+++ b/module/zcommon/zfs_prop.c
@@ -52,7 +52,11 @@ const char *zfs_userquota_prop_prefixes[] = {
 	"userused@",
 	"userquota@",
 	"groupused@",
-	"groupquota@"
+	"groupquota@",
+	"userobjused@",
+	"userobjquota@",
+	"groupobjused@",
+	"groupobjquota@"
 };
 
 zprop_desc_t *
diff --git a/module/zfs/dmu_objset.c b/module/zfs/dmu_objset.c
index 970ee4f08..c6b6eb745 100644
--- a/module/zfs/dmu_objset.c
+++ b/module/zfs/dmu_objset.c
@@ -31,6 +31,7 @@
 
 /* Portions Copyright 2010 Robert Milkowski */
 
+#include <sys/zfeature.h>
 #include <sys/cred.h>
 #include <sys/zfs_context.h>
 #include <sys/dmu_objset.h>
@@ -53,6 +54,7 @@
 #include <sys/dsl_destroy.h>
 #include <sys/vdev.h>
 #include <sys/policy.h>
+#include <sys/spa_impl.h>
 
 /*
  * Needed to close a window in dnode_move() that allows the objset to be freed
@@ -77,6 +79,9 @@ int dmu_rescan_dnode_threshold = 1 << DN_MAX_INDBLKSHIFT;
 
 static void dmu_objset_find_dp_cb(void *arg);
 
+static void dmu_objset_upgrade(objset_t *os, dmu_objset_upgrade_cb_t cb);
+static void dmu_objset_upgrade_stop(objset_t *os);
+
 void
 dmu_objset_init(void)
 {
@@ -519,6 +524,8 @@ dmu_objset_open_impl(spa_t *spa, dsl_dataset_t *ds, blkptr_t *bp,
 		    DMU_GROUPUSED_OBJECT, &os->os_groupused_dnode);
 	}
 
+	mutex_init(&os->os_upgrade_lock, NULL, MUTEX_DEFAULT, NULL);
+
 	*osp = os;
 	return (0);
 }
@@ -625,6 +632,9 @@ dmu_objset_own(const char *name, dmu_objset_type_t type,
 	err = dmu_objset_own_impl(ds, type, readonly, tag, osp);
 	dsl_pool_rele(dp, FTAG);
 
+	if (err == 0 && dmu_objset_userobjspace_upgradable(*osp))
+		dmu_objset_userobjspace_upgrade(*osp);
+
 	return (err);
 }
 
@@ -685,6 +695,10 @@ dmu_objset_refresh_ownership(objset_t *os, void *tag)
 void
 dmu_objset_disown(objset_t *os, void *tag)
 {
+	/*
+	 * Stop upgrading thread
+	 */
+	dmu_objset_upgrade_stop(os);
 	dsl_dataset_disown(os->os_dsl_dataset, tag);
 }
 
@@ -859,6 +873,12 @@ dmu_objset_create_impl(spa_t *spa, dsl_dataset_t *ds, blkptr_t *bp,
 	os->os_phys->os_type = type;
 	if (dmu_objset_userused_enabled(os)) {
 		os->os_phys->os_flags |= OBJSET_FLAG_USERACCOUNTING_COMPLETE;
+		if (dmu_objset_userobjused_enabled(os)) {
+			ds->ds_feature_activation_needed[
+			    SPA_FEATURE_USEROBJ_ACCOUNTING] = B_TRUE;
+			os->os_phys->os_flags |=
+			    OBJSET_FLAG_USEROBJACCOUNTING_COMPLETE;
+		}
 		os->os_flags = os->os_phys->os_flags;
 	}
 
@@ -1068,6 +1088,60 @@ dmu_objset_snapshot_one(const char *fsname, const char *snapname)
 }
 
 static void
+dmu_objset_upgrade_task_cb(void *data)
+{
+	objset_t *os = data;
+
+	mutex_enter(&os->os_upgrade_lock);
+	os->os_upgrade_status = EINTR;
+	if (!os->os_upgrade_exit) {
+		mutex_exit(&os->os_upgrade_lock);
+
+		os->os_upgrade_status = os->os_upgrade_cb(os);
+		mutex_enter(&os->os_upgrade_lock);
+	}
+	os->os_upgrade_exit = B_TRUE;
+	os->os_upgrade_id = 0;
+	mutex_exit(&os->os_upgrade_lock);
+}
+
+static void
+dmu_objset_upgrade(objset_t *os, dmu_objset_upgrade_cb_t cb)
+{
+	if (os->os_upgrade_id != 0)
+		return;
+
+	mutex_enter(&os->os_upgrade_lock);
+	if (os->os_upgrade_id == 0 && os->os_upgrade_status == 0) {
+		os->os_upgrade_exit = B_FALSE;
+		os->os_upgrade_cb = cb;
+		os->os_upgrade_id = taskq_dispatch(
+		    os->os_spa->spa_upgrade_taskq,
+		    dmu_objset_upgrade_task_cb, os, TQ_SLEEP);
+		if (os->os_upgrade_id == 0)
+			os->os_upgrade_status = ENOMEM;
+	}
+	mutex_exit(&os->os_upgrade_lock);
+}
+
+static void
+dmu_objset_upgrade_stop(objset_t *os)
+{
+	mutex_enter(&os->os_upgrade_lock);
+	os->os_upgrade_exit = B_TRUE;
+	if (os->os_upgrade_id != 0) {
+		taskqid_t id = os->os_upgrade_id;
+
+		os->os_upgrade_id = 0;
+		mutex_exit(&os->os_upgrade_lock);
+
+		taskq_cancel_id(os->os_spa->spa_upgrade_taskq, id);
+	} else {
+		mutex_exit(&os->os_upgrade_lock);
+	}
+}
+
+static void
 dmu_objset_sync_dnodes(list_t *list, list_t *newlist, dmu_tx_t *tx)
 {
 	dnode_t *dn;
@@ -1257,6 +1331,13 @@ dmu_objset_userused_enabled(objset_t *os)
 	    DMU_USERUSED_DNODE(os) != NULL);
 }
 
+boolean_t
+dmu_objset_userobjused_enabled(objset_t *os)
+{
+	return (dmu_objset_userused_enabled(os) &&
+	    spa_feature_is_enabled(os->os_spa, SPA_FEATURE_USEROBJ_ACCOUNTING));
+}
+
 static void
 do_userquota_update(objset_t *os, uint64_t used, uint64_t flags,
     uint64_t user, uint64_t group, boolean_t subtract, dmu_tx_t *tx)
@@ -1272,6 +1353,25 @@ do_userquota_update(objset_t *os, uint64_t used, uint64_t flags,
 	}
 }
 
+static void
+do_userobjquota_update(objset_t *os, uint64_t flags, uint64_t user,
+    uint64_t group, boolean_t subtract, dmu_tx_t *tx)
+{
+	if (flags & DNODE_FLAG_USEROBJUSED_ACCOUNTED) {
+		char name[20 + DMU_OBJACCT_PREFIX_LEN];
+
+		(void) snprintf(name, sizeof (name), DMU_OBJACCT_PREFIX "%llx",
+		    (longlong_t)user);
+		VERIFY0(zap_increment(os, DMU_USERUSED_OBJECT, name,
+		    subtract ? -1 : 1, tx));
+
+		(void) snprintf(name, sizeof (name), DMU_OBJACCT_PREFIX "%llx",
+		    (longlong_t)group);
+		VERIFY0(zap_increment(os, DMU_GROUPUSED_OBJECT, name,
+		    subtract ? -1 : 1, tx));
+	}
+}
+
 void
 dmu_objset_do_userquota_updates(objset_t *os, dmu_tx_t *tx)
 {
@@ -1310,11 +1410,15 @@ dmu_objset_do_userquota_updates(objset_t *os, dmu_tx_t *tx)
 		if (flags & DN_ID_OLD_EXIST)  {
 			do_userquota_update(os, dn->dn_oldused, dn->dn_oldflags,
 			    dn->dn_olduid, dn->dn_oldgid, B_TRUE, tx);
+			do_userobjquota_update(os, dn->dn_oldflags,
+			    dn->dn_olduid, dn->dn_oldgid, B_TRUE, tx);
 		}
 		if (flags & DN_ID_NEW_EXIST) {
 			do_userquota_update(os, DN_USED_BYTES(dn->dn_phys),
 			    dn->dn_phys->dn_flags,  dn->dn_newuid,
 			    dn->dn_newgid, B_FALSE, tx);
+			do_userobjquota_update(os, dn->dn_phys->dn_flags,
+			    dn->dn_newuid, dn->dn_newgid, B_FALSE, tx);
 		}
 
 		mutex_enter(&dn->dn_mtx);
@@ -1486,19 +1590,19 @@ dmu_objset_userspace_present(objset_t *os)
 	    OBJSET_FLAG_USERACCOUNTING_COMPLETE);
 }
 
-int
-dmu_objset_userspace_upgrade(objset_t *os)
+boolean_t
+dmu_objset_userobjspace_present(objset_t *os)
+{
+	return (os->os_phys->os_flags &
+	    OBJSET_FLAG_USEROBJACCOUNTING_COMPLETE);
+}
+
+static int
+dmu_objset_space_upgrade(objset_t *os)
 {
 	uint64_t obj;
 	int err = 0;
 
-	if (dmu_objset_userspace_present(os))
-		return (0);
-	if (!dmu_objset_userused_enabled(os))
-		return (SET_ERROR(ENOTSUP));
-	if (dmu_objset_is_snapshot(os))
-		return (SET_ERROR(EINVAL));
-
 	/*
 	 * We simply need to mark every object dirty, so that it will be
 	 * synced out and now accounted.  If this is called
@@ -1512,6 +1616,13 @@ dmu_objset_userspace_upgrade(objset_t *os)
 		dmu_buf_t *db;
 		int objerr;
 
+		mutex_enter(&os->os_upgrade_lock);
+		if (os->os_upgrade_exit)
+			err = SET_ERROR(EINTR);
+		mutex_exit(&os->os_upgrade_lock);
+		if (err != 0)
+			return (err);
+
 		if (issig(JUSTLOOKING) && issig(FORREAL))
 			return (SET_ERROR(EINTR));
 
@@ -1529,12 +1640,60 @@ dmu_objset_userspace_upgrade(objset_t *os)
 		dmu_buf_rele(db, FTAG);
 		dmu_tx_commit(tx);
 	}
+	return (0);
+}
+
+int
+dmu_objset_userspace_upgrade(objset_t *os)
+{
+	int err = 0;
+
+	if (dmu_objset_userspace_present(os))
+		return (0);
+	if (dmu_objset_is_snapshot(os))
+		return (SET_ERROR(EINVAL));
+	if (!dmu_objset_userused_enabled(os))
+		return (SET_ERROR(ENOTSUP));
+
+	err = dmu_objset_space_upgrade(os);
+	if (err)
+		return (err);
 
 	os->os_flags |= OBJSET_FLAG_USERACCOUNTING_COMPLETE;
 	txg_wait_synced(dmu_objset_pool(os), 0);
 	return (0);
 }
 
+static int
+dmu_objset_userobjspace_upgrade_cb(objset_t *os)
+{
+	int err = 0;
+
+	if (dmu_objset_userobjspace_present(os))
+		return (0);
+	if (dmu_objset_is_snapshot(os))
+		return (SET_ERROR(EINVAL));
+	if (!dmu_objset_userobjused_enabled(os))
+		return (SET_ERROR(ENOTSUP));
+
+	dmu_objset_ds(os)->ds_feature_activation_needed[
+	    SPA_FEATURE_USEROBJ_ACCOUNTING] = B_TRUE;
+
+	err = dmu_objset_space_upgrade(os);
+	if (err)
+		return (err);
+
+	os->os_flags |= OBJSET_FLAG_USEROBJACCOUNTING_COMPLETE;
+	txg_wait_synced(dmu_objset_pool(os), 0);
+	return (0);
+}
+
+void
+dmu_objset_userobjspace_upgrade(objset_t *os)
+{
+	dmu_objset_upgrade(os, dmu_objset_userobjspace_upgrade_cb);
+}
+
 void
 dmu_objset_space(objset_t *os, uint64_t *refdbytesp, uint64_t *availbytesp,
     uint64_t *usedobjsp, uint64_t *availobjsp)
@@ -2096,4 +2255,7 @@ EXPORT_SYMBOL(dmu_objset_userquota_get_ids);
 EXPORT_SYMBOL(dmu_objset_userused_enabled);
 EXPORT_SYMBOL(dmu_objset_userspace_upgrade);
 EXPORT_SYMBOL(dmu_objset_userspace_present);
+EXPORT_SYMBOL(dmu_objset_userobjused_enabled);
+EXPORT_SYMBOL(dmu_objset_userobjspace_upgrade);
+EXPORT_SYMBOL(dmu_objset_userobjspace_present);
 #endif
diff --git a/module/zfs/dnode_sync.c b/module/zfs/dnode_sync.c
index b19f50af9..6d1fa3339 100644
--- a/module/zfs/dnode_sync.c
+++ b/module/zfs/dnode_sync.c
@@ -570,12 +570,17 @@ dnode_sync(dnode_t *dn, dmu_tx_t *tx)
 		dn->dn_oldused = DN_USED_BYTES(dn->dn_phys);
 		dn->dn_oldflags = dn->dn_phys->dn_flags;
 		dn->dn_phys->dn_flags |= DNODE_FLAG_USERUSED_ACCOUNTED;
+		if (dmu_objset_userobjused_enabled(dn->dn_objset))
+			dn->dn_phys->dn_flags |=
+			    DNODE_FLAG_USEROBJUSED_ACCOUNTED;
 		mutex_exit(&dn->dn_mtx);
 		dmu_objset_userquota_get_ids(dn, B_FALSE, tx);
 	} else {
 		/* Once we account for it, we should always account for it. */
 		ASSERT(!(dn->dn_phys->dn_flags &
 		    DNODE_FLAG_USERUSED_ACCOUNTED));
+		ASSERT(!(dn->dn_phys->dn_flags &
+		    DNODE_FLAG_USEROBJUSED_ACCOUNTED));
 	}
 
 	mutex_enter(&dn->dn_mtx);
diff --git a/module/zfs/spa.c b/module/zfs/spa.c
index c2f914e11..0a480d3ec 100644
--- a/module/zfs/spa.c
+++ b/module/zfs/spa.c
@@ -1167,6 +1167,13 @@ spa_activate(spa_t *spa, int mode)
 	 */
 	spa->spa_zvol_taskq = taskq_create("z_zvol", 1, defclsyspri,
 	    1, INT_MAX, 0);
+
+	/*
+	 * The taskq to upgrade datasets in this pool. Currently used by
+	 * feature SPA_FEATURE_USEROBJ_ACCOUNTING.
+	 */
+	spa->spa_upgrade_taskq = taskq_create("z_upgrade", boot_ncpus,
+	    defclsyspri, 1, INT_MAX, TASKQ_DYNAMIC);
 }
 
 /*
@@ -1190,6 +1197,11 @@ spa_deactivate(spa_t *spa)
 		spa->spa_zvol_taskq = NULL;
 	}
 
+	if (spa->spa_upgrade_taskq) {
+		taskq_destroy(spa->spa_upgrade_taskq);
+		spa->spa_upgrade_taskq = NULL;
+	}
+
 	txg_list_destroy(&spa->spa_vdev_txg_list);
 
 	list_destroy(&spa->spa_config_dirty_list);
diff --git a/module/zfs/zfeature_common.c b/module/zfs/zfeature_common.c
index 9beb4903e..ccd65a7b7 100644
--- a/module/zfs/zfeature_common.c
+++ b/module/zfs/zfeature_common.c
@@ -285,4 +285,15 @@ zpool_feature_init(void)
 	    "Edon-R hash algorithm.",
 	    ZFEATURE_FLAG_PER_DATASET, edonr_deps);
 	}
+	{
+	static const spa_feature_t userobj_accounting_deps[] = {
+		SPA_FEATURE_EXTENSIBLE_DATASET,
+		SPA_FEATURE_NONE
+	};
+	zfeature_register(SPA_FEATURE_USEROBJ_ACCOUNTING,
+	    "org.zfsonlinux:userobj_accounting", "userobj_accounting",
+	    "User/Group object accounting.",
+	    ZFEATURE_FLAG_READONLY_COMPAT | ZFEATURE_FLAG_PER_DATASET,
+	    userobj_accounting_deps);
+	}
 }
diff --git a/module/zfs/zfs_acl.c b/module/zfs/zfs_acl.c
index 451000010..7198c7ebf 100644
--- a/module/zfs/zfs_acl.c
+++ b/module/zfs/zfs_acl.c
@@ -1886,7 +1886,9 @@ boolean_t
 zfs_acl_ids_overquota(zfs_sb_t *zsb, zfs_acl_ids_t *acl_ids)
 {
 	return (zfs_fuid_overquota(zsb, B_FALSE, acl_ids->z_fuid) ||
-	    zfs_fuid_overquota(zsb, B_TRUE, acl_ids->z_fgid));
+	    zfs_fuid_overquota(zsb, B_TRUE, acl_ids->z_fgid) ||
+	    zfs_fuid_overobjquota(zsb, B_FALSE, acl_ids->z_fuid) ||
+	    zfs_fuid_overobjquota(zsb, B_TRUE, acl_ids->z_fgid));
 }
 
 /*
diff --git a/module/zfs/zfs_ioctl.c b/module/zfs/zfs_ioctl.c
index e5704e258..549a83116 100644
--- a/module/zfs/zfs_ioctl.c
+++ b/module/zfs/zfs_ioctl.c
@@ -244,9 +244,14 @@ static const char *userquota_perms[] = {
 	ZFS_DELEG_PERM_USERQUOTA,
 	ZFS_DELEG_PERM_GROUPUSED,
 	ZFS_DELEG_PERM_GROUPQUOTA,
+	ZFS_DELEG_PERM_USEROBJUSED,
+	ZFS_DELEG_PERM_USEROBJQUOTA,
+	ZFS_DELEG_PERM_GROUPOBJUSED,
+	ZFS_DELEG_PERM_GROUPOBJQUOTA,
 };
 
 static int zfs_ioc_userspace_upgrade(zfs_cmd_t *zc);
+static int zfs_ioc_userobjspace_upgrade(zfs_cmd_t *zc);
 static int zfs_check_settable(const char *name, nvpair_t *property,
     cred_t *cr);
 static int zfs_check_clearable(char *dataset, nvlist_t *props,
@@ -1171,7 +1176,9 @@ zfs_secpolicy_userspace_one(zfs_cmd_t *zc, nvlist_t *innvl, cred_t *cr)
 		 * themself, allow it.
 		 */
 		if (zc->zc_objset_type == ZFS_PROP_USERUSED ||
-		    zc->zc_objset_type == ZFS_PROP_USERQUOTA) {
+		    zc->zc_objset_type == ZFS_PROP_USERQUOTA ||
+		    zc->zc_objset_type == ZFS_PROP_USEROBJUSED ||
+		    zc->zc_objset_type == ZFS_PROP_USEROBJQUOTA) {
 			if (zc->zc_guid == crgetuid(cr))
 				return (0);
 		} else {
@@ -2426,6 +2433,7 @@ zfs_prop_set_special(const char *dsname, zprop_source_t source,
 			zc = kmem_zalloc(sizeof (zfs_cmd_t), KM_SLEEP);
 			(void) strcpy(zc->zc_name, dsname);
 			(void) zfs_ioc_userspace_upgrade(zc);
+			(void) zfs_ioc_userobjspace_upgrade(zc);
 			kmem_free(zc, sizeof (zfs_cmd_t));
 		}
 		break;
@@ -3720,13 +3728,23 @@ zfs_check_settable(const char *dsname, nvpair_t *pair, cred_t *cr)
 			    zfs_userquota_prop_prefixes[ZFS_PROP_USERQUOTA];
 			const char *gq_prefix =
 			    zfs_userquota_prop_prefixes[ZFS_PROP_GROUPQUOTA];
+			const char *uiq_prefix =
+			    zfs_userquota_prop_prefixes[ZFS_PROP_USEROBJQUOTA];
+			const char *giq_prefix =
+			    zfs_userquota_prop_prefixes[ZFS_PROP_GROUPOBJQUOTA];
 
 			if (strncmp(propname, uq_prefix,
 			    strlen(uq_prefix)) == 0) {
 				perm = ZFS_DELEG_PERM_USERQUOTA;
+			} else if (strncmp(propname, uiq_prefix,
+			    strlen(uiq_prefix)) == 0) {
+				perm = ZFS_DELEG_PERM_USEROBJQUOTA;
 			} else if (strncmp(propname, gq_prefix,
 			    strlen(gq_prefix)) == 0) {
 				perm = ZFS_DELEG_PERM_GROUPQUOTA;
+			} else if (strncmp(propname, giq_prefix,
+			    strlen(giq_prefix)) == 0) {
+				perm = ZFS_DELEG_PERM_GROUPOBJQUOTA;
 			} else {
 				/* USERUSED and GROUPUSED are read-only */
 				return (SET_ERROR(EINVAL));
@@ -4927,6 +4945,48 @@ zfs_ioc_userspace_upgrade(zfs_cmd_t *zc)
 	return (error);
 }
 
+/*
+ * inputs:
+ * zc_name		name of filesystem
+ *
+ * outputs:
+ * none
+ */
+static int
+zfs_ioc_userobjspace_upgrade(zfs_cmd_t *zc)
+{
+	objset_t *os;
+	int error;
+
+	error = dmu_objset_hold(zc->zc_name, FTAG, &os);
+	if (error != 0)
+		return (error);
+
+	dsl_dataset_long_hold(dmu_objset_ds(os), FTAG);
+	dsl_pool_rele(dmu_objset_pool(os), FTAG);
+
+	if (dmu_objset_userobjspace_upgradable(os)) {
+		mutex_enter(&os->os_upgrade_lock);
+		if (os->os_upgrade_id == 0) {
+			/* clear potential error code and retry */
+			os->os_upgrade_status = 0;
+			mutex_exit(&os->os_upgrade_lock);
+
+			dmu_objset_userobjspace_upgrade(os);
+		} else {
+			mutex_exit(&os->os_upgrade_lock);
+		}
+
+		taskq_wait_id(os->os_spa->spa_upgrade_taskq, os->os_upgrade_id);
+		error = os->os_upgrade_status;
+	}
+
+	dsl_dataset_long_rele(dmu_objset_ds(os), FTAG);
+	dsl_dataset_rele(dmu_objset_ds(os), FTAG);
+
+	return (error);
+}
+
 static int
 zfs_ioc_share(zfs_cmd_t *zc)
 {
diff --git a/module/zfs/zfs_vfsops.c b/module/zfs/zfs_vfsops.c
index d8b27461a..63b7f9230 100644
--- a/module/zfs/zfs_vfsops.c
+++ b/module/zfs/zfs_vfsops.c
@@ -431,17 +431,22 @@ zfs_userquota_prop_to_obj(zfs_sb_t *zsb, zfs_userquota_prop_t type)
 {
 	switch (type) {
 	case ZFS_PROP_USERUSED:
+	case ZFS_PROP_USEROBJUSED:
 		return (DMU_USERUSED_OBJECT);
 	case ZFS_PROP_GROUPUSED:
+	case ZFS_PROP_GROUPOBJUSED:
 		return (DMU_GROUPUSED_OBJECT);
 	case ZFS_PROP_USERQUOTA:
 		return (zsb->z_userquota_obj);
 	case ZFS_PROP_GROUPQUOTA:
 		return (zsb->z_groupquota_obj);
+	case ZFS_PROP_USEROBJQUOTA:
+		return (zsb->z_userobjquota_obj);
+	case ZFS_PROP_GROUPOBJQUOTA:
+		return (zsb->z_groupobjquota_obj);
 	default:
-		return (SET_ERROR(ENOTSUP));
+		return (ZFS_NO_OBJECT);
 	}
-	return (0);
 }
 
 int
@@ -453,16 +458,25 @@ zfs_userspace_many(zfs_sb_t *zsb, zfs_userquota_prop_t type,
 	zap_attribute_t za;
 	zfs_useracct_t *buf = vbuf;
 	uint64_t obj;
+	int offset = 0;
 
 	if (!dmu_objset_userspace_present(zsb->z_os))
 		return (SET_ERROR(ENOTSUP));
 
+	if ((type == ZFS_PROP_USEROBJUSED || type == ZFS_PROP_GROUPOBJUSED ||
+	    type == ZFS_PROP_USEROBJQUOTA || type == ZFS_PROP_GROUPOBJQUOTA) &&
+	    !dmu_objset_userobjspace_present(zsb->z_os))
+		return (SET_ERROR(ENOTSUP));
+
 	obj = zfs_userquota_prop_to_obj(zsb, type);
-	if (obj == 0) {
+	if (obj == ZFS_NO_OBJECT) {
 		*bufsizep = 0;
 		return (0);
 	}
 
+	if (type == ZFS_PROP_USEROBJUSED || type == ZFS_PROP_GROUPOBJUSED)
+		offset = DMU_OBJACCT_PREFIX_LEN;
+
 	for (zap_cursor_init_serialized(&zc, zsb->z_os, obj, *cookiep);
 	    (error = zap_cursor_retrieve(&zc, &za)) == 0;
 	    zap_cursor_advance(&zc)) {
@@ -470,7 +484,15 @@ zfs_userspace_many(zfs_sb_t *zsb, zfs_userquota_prop_t type,
 		    *bufsizep)
 			break;
 
-		fuidstr_to_sid(zsb, za.za_name,
+		/*
+		 * skip object quota (with zap name prefix DMU_OBJACCT_PREFIX)
+		 * when dealing with block quota and vice versa.
+		 */
+		if ((offset > 0) != (strncmp(za.za_name, DMU_OBJACCT_PREFIX,
+		    DMU_OBJACCT_PREFIX_LEN) == 0))
+			continue;
+
+		fuidstr_to_sid(zsb, za.za_name + offset,
 		    buf->zu_domain, sizeof (buf->zu_domain), &buf->zu_rid);
 
 		buf->zu_space = za.za_first_integer;
@@ -511,7 +533,8 @@ int
 zfs_userspace_one(zfs_sb_t *zsb, zfs_userquota_prop_t type,
     const char *domain, uint64_t rid, uint64_t *valp)
 {
-	char buf[32];
+	char buf[20 + DMU_OBJACCT_PREFIX_LEN];
+	int offset = 0;
 	int err;
 	uint64_t obj;
 
@@ -520,11 +543,21 @@ zfs_userspace_one(zfs_sb_t *zsb, zfs_userquota_prop_t type,
 	if (!dmu_objset_userspace_present(zsb->z_os))
 		return (SET_ERROR(ENOTSUP));
 
+	if ((type == ZFS_PROP_USEROBJUSED || type == ZFS_PROP_GROUPOBJUSED ||
+	    type == ZFS_PROP_USEROBJQUOTA || type == ZFS_PROP_GROUPOBJQUOTA) &&
+	    !dmu_objset_userobjspace_present(zsb->z_os))
+		return (SET_ERROR(ENOTSUP));
+
 	obj = zfs_userquota_prop_to_obj(zsb, type);
-	if (obj == 0)
+	if (obj == ZFS_NO_OBJECT)
 		return (0);
 
-	err = id_to_fuidstr(zsb, domain, rid, buf, B_FALSE);
+	if (type == ZFS_PROP_USEROBJUSED || type == ZFS_PROP_GROUPOBJUSED) {
+		strncpy(buf, DMU_OBJACCT_PREFIX, DMU_OBJACCT_PREFIX_LEN);
+		offset = DMU_OBJACCT_PREFIX_LEN;
+	}
+
+	err = id_to_fuidstr(zsb, domain, rid, buf + offset, B_FALSE);
 	if (err)
 		return (err);
 
@@ -545,14 +578,25 @@ zfs_set_userquota(zfs_sb_t *zsb, zfs_userquota_prop_t type,
 	uint64_t *objp;
 	boolean_t fuid_dirtied;
 
-	if (type != ZFS_PROP_USERQUOTA && type != ZFS_PROP_GROUPQUOTA)
-		return (SET_ERROR(EINVAL));
-
 	if (zsb->z_version < ZPL_VERSION_USERSPACE)
 		return (SET_ERROR(ENOTSUP));
 
-	objp = (type == ZFS_PROP_USERQUOTA) ? &zsb->z_userquota_obj :
-	    &zsb->z_groupquota_obj;
+	switch (type) {
+	case ZFS_PROP_USERQUOTA:
+		objp = &zsb->z_userquota_obj;
+		break;
+	case ZFS_PROP_GROUPQUOTA:
+		objp = &zsb->z_groupquota_obj;
+		break;
+	case ZFS_PROP_USEROBJQUOTA:
+		objp = &zsb->z_userobjquota_obj;
+		break;
+	case ZFS_PROP_GROUPOBJQUOTA:
+		objp = &zsb->z_groupobjquota_obj;
+		break;
+	default:
+		return (SET_ERROR(EINVAL));
+	}
 
 	err = id_to_fuidstr(zsb, domain, rid, buf, B_TRUE);
 	if (err)
@@ -598,9 +642,39 @@ zfs_set_userquota(zfs_sb_t *zsb, zfs_userquota_prop_t type,
 EXPORT_SYMBOL(zfs_set_userquota);
 
 boolean_t
+zfs_fuid_overobjquota(zfs_sb_t *zsb, boolean_t isgroup, uint64_t fuid)
+{
+	char buf[20 + DMU_OBJACCT_PREFIX_LEN];
+	uint64_t used, quota, usedobj, quotaobj;
+	int err;
+
+	if (!dmu_objset_userobjspace_present(zsb->z_os)) {
+		if (dmu_objset_userobjspace_upgradable(zsb->z_os))
+			dmu_objset_userobjspace_upgrade(zsb->z_os);
+		return (B_FALSE);
+	}
+
+	usedobj = isgroup ? DMU_GROUPUSED_OBJECT : DMU_USERUSED_OBJECT;
+	quotaobj = isgroup ? zsb->z_groupobjquota_obj : zsb->z_userobjquota_obj;
+	if (quotaobj == 0 || zsb->z_replay)
+		return (B_FALSE);
+
+	(void) sprintf(buf, "%llx", (longlong_t)fuid);
+	err = zap_lookup(zsb->z_os, quotaobj, buf, 8, 1, &quota);
+	if (err != 0)
+		return (B_FALSE);
+
+	(void) sprintf(buf, DMU_OBJACCT_PREFIX "%llx", (longlong_t)fuid);
+	err = zap_lookup(zsb->z_os, usedobj, buf, 8, 1, &used);
+	if (err != 0)
+		return (B_FALSE);
+	return (used >= quota);
+}
+
+boolean_t
 zfs_fuid_overquota(zfs_sb_t *zsb, boolean_t isgroup, uint64_t fuid)
 {
-	char buf[32];
+	char buf[20];
 	uint64_t used, quota, usedobj, quotaobj;
 	int err;
 
@@ -777,6 +851,18 @@ zfs_sb_create(const char *osname, zfs_mntopts_t *zmo, zfs_sb_t **zsbp)
 	if (error && error != ENOENT)
 		goto out;
 
+	error = zap_lookup(os, MASTER_NODE_OBJ,
+	    zfs_userquota_prop_prefixes[ZFS_PROP_USEROBJQUOTA],
+	    8, 1, &zsb->z_userobjquota_obj);
+	if (error && error != ENOENT)
+		goto out;
+
+	error = zap_lookup(os, MASTER_NODE_OBJ,
+	    zfs_userquota_prop_prefixes[ZFS_PROP_GROUPOBJQUOTA],
+	    8, 1, &zsb->z_groupobjquota_obj);
+	if (error && error != ENOENT)
+		goto out;
+
 	error = zap_lookup(os, MASTER_NODE_OBJ, ZFS_FUID_TABLES, 8, 1,
 	    &zsb->z_fuid_obj);
 	if (error && error != ENOENT)
diff --git a/tests/runfiles/linux.run b/tests/runfiles/linux.run
index 92f867ab9..f707205b3 100644
--- a/tests/runfiles/linux.run
+++ b/tests/runfiles/linux.run
@@ -602,21 +602,18 @@ tests = ['sparse_001_pos']
 [tests/functional/truncate]
 tests = ['truncate_001_pos', 'truncate_002_pos']
 
-# DISABLED:
-# groupspace_001_pos
-# groupspace_002_pos
-# userquota_001_pos
-# userquota_004_pos
-# userquota_007_pos
-# userquota_010_pos
-# userspace_001_pos
-# userspace_002_pos
+[tests/functional/upgrade]
+tests = [ 'upgrade_userobj_001_pos' ]
+
 [tests/functional/userquota]
 tests = [
-    'userquota_002_pos', 'userquota_003_pos',
-    'userquota_005_neg', 'userquota_006_pos',
-    'userquota_008_pos', 'userquota_009_pos',
-    'userquota_011_pos', 'userquota_012_neg']
+    'userquota_001_pos', 'userquota_002_pos', 'userquota_003_pos',
+    'userquota_004_pos', 'userquota_005_neg', 'userquota_006_pos',
+    'userquota_007_pos', 'userquota_008_pos', 'userquota_009_pos',
+    'userquota_010_pos', 'userquota_011_pos', 'userquota_012_neg',
+    'userquota_013_pos',
+    'userspace_001_pos', 'userspace_002_pos', 'userspace_003_pos',
+    'groupspace_001_pos', 'groupspace_002_pos', 'groupspace_003_pos' ]
 
 # DISABLED:
 # vdev_zaps_007_pos -- fails due to a pre-existing issue with zpool split
diff --git a/tests/zfs-tests/cmd/mkfiles/mkfiles.c b/tests/zfs-tests/cmd/mkfiles/mkfiles.c
index 418fb9d07..62dee1627 100644
--- a/tests/zfs-tests/cmd/mkfiles/mkfiles.c
+++ b/tests/zfs-tests/cmd/mkfiles/mkfiles.c
@@ -48,10 +48,7 @@ main(int argc, char **argv)
 	if (argc == 4 && sscanf(argv[3], "%u", &first_file) != 1)
 		usage("Invalid first file", -3);
 
-	if (numfiles < first_file)
-		usage("First file larger than last file", -3);
-
-	for (i = first_file; i <= numfiles; i++) {
+	for (i = first_file; i < first_file + numfiles; i++) {
 		int fd;
 		(void) snprintf(buf, MAXPATHLEN, "%s%u", argv[1], i);
 		if ((fd = open(buf, O_CREAT | O_EXCL, O_RDWR)) == -1) {
diff --git a/tests/zfs-tests/include/libtest.shlib b/tests/zfs-tests/include/libtest.shlib
index 62ba3a9eb..1857cf91f 100644
--- a/tests/zfs-tests/include/libtest.shlib
+++ b/tests/zfs-tests/include/libtest.shlib
@@ -2857,3 +2857,29 @@ function block_device_wait
 		$UDEVADM settle
 	fi
 }
+
+#
+# Synchronize all the data in pool
+#
+# $1 pool name
+#
+function sync_pool #pool
+{
+	typeset pool=${1:-$TESTPOOL}
+
+	log_must $SYNC
+	log_must $SLEEP 2
+	# Flush all the pool data.
+	typeset -i ret
+	$ZPOOL scrub $pool >/dev/null 2>&1
+	ret=$?
+	(( $ret != 0 )) && \
+	log_fail "$ZPOOL scrub $pool failed."
+
+	while ! is_pool_scrubbed $pool; do
+		if is_pool_resilvered $pool ; then
+			log_fail "$pool should not be resilver completed."
+		fi
+		log_must $SLEEP 2
+	done
+}
diff --git a/tests/zfs-tests/tests/functional/Makefile.am b/tests/zfs-tests/tests/functional/Makefile.am
index ed01eafb4..ffba71b51 100644
--- a/tests/zfs-tests/tests/functional/Makefile.am
+++ b/tests/zfs-tests/tests/functional/Makefile.am
@@ -51,6 +51,7 @@ SUBDIRS = \
 	sparse \
 	threadsappend \
 	truncate \
+	upgrade \
 	userquota \
 	vdev_zaps \
 	write_dirs \
diff --git a/tests/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg b/tests/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg
index 3807d0af6..699229fef 100644
--- a/tests/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg
+++ b/tests/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg
@@ -38,7 +38,8 @@ typeset -a properties=("size" "capacity" "altroot" "health" "guid" "version"
     "feature@large_blocks" "feature@large_dnode" "feature@filesystem_limits"
     "feature@spacemap_histogram" "feature@enabled_txg" "feature@hole_birth"
     "feature@extensible_dataset" "feature@bookmarks" "feature@embedded_data"
-    "feature@sha512" "feature@skein" "feature@edonr")
+    "feature@sha512" "feature@skein" "feature@edonr"
+    "feature@userobj_accounting")
 else
 typeset -a properties=("size" "capacity" "altroot" "health" "guid" "version"
     "bootfs" ""leaked" delegation" "autoreplace" "cachefile" "dedupditto" "dedupratio"
diff --git a/tests/zfs-tests/tests/functional/redundancy/redundancy.kshlib b/tests/zfs-tests/tests/functional/redundancy/redundancy.kshlib
index cb8271797..56e2bd19d 100644
--- a/tests/zfs-tests/tests/functional/redundancy/redundancy.kshlib
+++ b/tests/zfs-tests/tests/functional/redundancy/redundancy.kshlib
@@ -214,32 +214,6 @@ function get_vdevs #pool cnt
 }
 
 #
-# Synchronize all the data in pool
-#
-# $1 pool name
-#
-function sync_pool #pool
-{
-	typeset pool=$1
-
-	log_must $SYNC
-	log_must $SLEEP 2
-	# Flush all the pool data.
-	typeset -i ret
-	$ZPOOL scrub $pool >/dev/null 2>&1
-	ret=$?
-	(( $ret != 0 )) && \
-		log_fail "$ZPOOL scrub $pool failed."
-
-	while ! is_pool_scrubbed $pool; do
-		if is_pool_resilvered $pool ; then
-			log_fail "$pool should not be resilver completed."
-		fi
-		log_must $SLEEP 2
-	done
-}
-
-#
 # Create and replace the same name virtual device files
 #
 # $1 pool name
diff --git a/tests/zfs-tests/tests/functional/upgrade/Makefile.am b/tests/zfs-tests/tests/functional/upgrade/Makefile.am
new file mode 100644
index 000000000..31034342f
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/upgrade/Makefile.am
@@ -0,0 +1,5 @@
+pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/upgrade
+dist_pkgdata_SCRIPTS = \
+	setup.ksh \
+	cleanup.ksh \
+	upgrade_userobj_001_pos.ksh
diff --git a/tests/zfs-tests/tests/functional/upgrade/cleanup.ksh b/tests/zfs-tests/tests/functional/upgrade/cleanup.ksh
new file mode 100644
index 000000000..6b0eb9d9a
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/upgrade/cleanup.ksh
@@ -0,0 +1,44 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+# Use is subject to license terms.
+#
+
+#
+# Copyright (c) 2013 by Delphix. All rights reserved.
+#
+
+#
+# Copyright (c) 2016 by Jinshan Xiong. No rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+verify_runnable "global"
+
+log_must $ZPOOL destroy $TESTPOOL
+
+log_must $RM /tmp/zpool_upgrade_test.dat
+
+default_cleanup
diff --git a/tests/zfs-tests/tests/functional/upgrade/setup.ksh b/tests/zfs-tests/tests/functional/upgrade/setup.ksh
new file mode 100644
index 000000000..57b483581
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/upgrade/setup.ksh
@@ -0,0 +1,44 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+# Use is subject to license terms.
+#
+
+#
+# Copyright (c) 2013 by Delphix. All rights reserved.
+#
+
+#
+# Copyright (c) 2016 by Jinshan Xiong. No rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+verify_runnable "global"
+
+# create a pool without any features
+log_must $MKFILE 128m /tmp/zpool_upgrade_test.dat
+log_must $ZPOOL create -d -m $TESTDIR $TESTPOOL /tmp/zpool_upgrade_test.dat
+
+log_pass
diff --git a/tests/zfs-tests/tests/functional/upgrade/upgrade_userobj_001_pos.ksh b/tests/zfs-tests/tests/functional/upgrade/upgrade_userobj_001_pos.ksh
new file mode 100644
index 000000000..49087f5a1
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/upgrade/upgrade_userobj_001_pos.ksh
@@ -0,0 +1,98 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2013 by Jinshan Xiong. No rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+#
+# DESCRIPTION:
+#
+# Check that zfs upgrade for object count accounting works.
+# Since userobjaccounting is a per dataset feature, this test case
+# will create multiple dataset and try different upgrade method.
+#
+# STRATEGY:
+# 1. Create a pool with all features disabled
+# 2. Create a few dataset for testing
+# 3. Make sure automatic upgrade work
+# 4. Make sure manual upgrade work
+#
+
+function cleanup
+{
+	datasetexists $TESTPOOL/fs1 && log_must $ZFS destroy $TESTPOOL/fs1
+	datasetexists $TESTPOOL/fs2 && log_must $ZFS destroy $TESTPOOL/fs2
+}
+
+verify_runnable "global"
+
+log_assert "pool upgrade for userobj accounting should work"
+log_onexit cleanup
+
+log_must $MKFILES $TESTDIR/tf $((RANDOM % 1000 + 1))
+log_must $ZFS create $TESTPOOL/fs1
+log_must $MKFILES $TESTDIR/fs1/tf $((RANDOM % 1000 + 1))
+log_must $ZFS create $TESTPOOL/fs2
+log_must $MKFILES $TESTDIR/fs2/tf $((RANDOM % 1000 + 1))
+log_must $ZFS umount $TESTPOOL/fs2
+
+# Make sure userobj accounting is disabled
+$ZFS userspace -o objused -H $TESTPOOL | $HEAD -n 1 | $GREP -q "-" ||
+	log_fail "userobj accounting should be disabled initially"
+
+# Upgrade zpool to support all features
+log_must $ZPOOL upgrade $TESTPOOL
+
+# Make sure userobj accounting is disabled again
+$ZFS userspace -o objused -H $TESTPOOL | $HEAD -n 1 | $GREP -q "-" ||
+	log_fail "userobj accounting should be disabled after pool upgrade"
+
+# Create a file in fs1 should trigger dataset upgrade
+log_must $MKFILE 1m $TESTDIR/fs1/tf
+sync_pool
+
+# Make sure userobj accounting is working for fs1
+$ZFS userspace -o objused -H $TESTPOOL/fs1 | $HEAD -n 1 | $GREP -q "-" &&
+	log_fail "userobj accounting should be enabled for $TESTPOOL/fs1"
+
+# Mount a dataset should trigger upgrade
+log_must $ZFS mount $TESTPOOL/fs2
+sync_pool
+
+# Make sure userobj accounting is working for fs2
+$ZFS userspace -o objused -H $TESTPOOL/fs2 | $HEAD -n 1 | $GREP -q "-" &&
+	log_fail "userobj accounting should be enabled for $TESTPOOL/fs2"
+
+# All in all, after having been through this, the dataset for testpool
+# still shouldn't be upgraded
+$ZFS userspace -o objused -H $TESTPOOL | $HEAD -n 1 | $GREP -q "-" ||
+	log_fail "userobj accounting should be disabled for $TESTPOOL"
+
+# Manual upgrade root dataset
+log_must $ZFS set version=current $TESTPOOL
+$ZFS userspace -o objused -H $TESTPOOL | $HEAD -n 1 | $GREP -q "-" &&
+	log_fail "userobj accounting should be enabled for $TESTPOOL"
+
+log_pass "all tests passed - what a lucky day!"
diff --git a/tests/zfs-tests/tests/functional/userquota/Makefile.am b/tests/zfs-tests/tests/functional/userquota/Makefile.am
index b72659964..b7f38f98e 100644
--- a/tests/zfs-tests/tests/functional/userquota/Makefile.am
+++ b/tests/zfs-tests/tests/functional/userquota/Makefile.am
@@ -6,6 +6,7 @@ dist_pkgdata_SCRIPTS = \
 	cleanup.ksh \
 	groupspace_001_pos.ksh \
 	groupspace_002_pos.ksh \
+	groupspace_003_pos.ksh \
 	userquota_001_pos.ksh \
 	userquota_002_pos.ksh \
 	userquota_003_pos.ksh \
@@ -18,5 +19,7 @@ dist_pkgdata_SCRIPTS = \
 	userquota_010_pos.ksh \
 	userquota_011_pos.ksh \
 	userquota_012_neg.ksh \
+	userquota_013_pos.ksh \
 	userspace_001_pos.ksh \
-	userspace_002_pos.ksh
+	userspace_002_pos.ksh \
+	userspace_003_pos.ksh
diff --git a/tests/zfs-tests/tests/functional/userquota/groupspace_003_pos.ksh b/tests/zfs-tests/tests/functional/userquota/groupspace_003_pos.ksh
new file mode 100644
index 000000000..7ea8cd563
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/userquota/groupspace_003_pos.ksh
@@ -0,0 +1,103 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+# Use is subject to license terms.
+#
+
+#
+# Copyright (c) 2013 by Delphix. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/userquota/userquota_common.kshlib
+
+#
+# DESCRIPTION:
+#       Check the user used and groupspace object counts in zfs groupspace
+#
+#
+# STRATEGY:
+#	1. set zfs groupquota to a fs
+#	2. create objects for different users in the same group
+#	3. use zfs groupspace to check the object count
+#
+
+function cleanup
+{
+	if datasetexists $snapfs; then
+		log_must $ZFS destroy $snapfs
+	fi
+
+	log_must $RM -f ${QFILE}_*
+	log_must cleanup_quota
+}
+
+function group_object_count
+{
+	typeset fs=$1
+	typeset user=$2
+	typeset cnt=$($ZFS groupspace -oname,objused $fs | $GREP $user |
+			$AWK '{print $2}')
+	echo $cnt
+}
+
+log_onexit cleanup
+
+log_assert "Check the zfs groupspace object used"
+
+mkmount_writable $QFS
+log_must $ZFS set xattr=sa $QFS
+
+((user1_cnt = RANDOM % 100 + 1))
+((user2_cnt = RANDOM % 100 + 1))
+log_must user_run $QUSER1 $MKFILES ${QFILE}_1 $user1_cnt
+log_must user_run $QUSER2 $MKFILES ${QFILE}_2 $user2_cnt
+((grp_cnt = user1_cnt + user2_cnt))
+sync_pool
+
+typeset snapfs=$QFS@snap
+
+log_must $ZFS snapshot $snapfs
+
+log_must eval "$ZFS groupspace $QFS >/dev/null 2>&1"
+log_must eval "$ZFS groupspace $snapfs >/dev/null 2>&1"
+
+for fs in "$QFS" "$snapfs"; do
+	log_note "check the object count in zfs groupspace $fs"
+        [[ $(group_object_count $fs $QGROUP) -eq $grp_cnt ]] ||
+                log_fail "expected $grp_cnt"
+done
+
+log_note "file removal"
+log_must $RM ${QFILE}_*
+sync_pool
+
+[[ $(group_object_count $QFS $QGROUP) -eq 0 ]] ||
+        log_fail "expected 0 files for $QGROUP"
+
+[[ $(group_object_count $snapfs $QGROUP) -eq $grp_cnt ]] ||
+        log_fail "expected $grp_cnt files for $QGROUP"
+
+cleanup
+log_pass "Check the zfs groupspace object used pass as expect"
diff --git a/tests/zfs-tests/tests/functional/userquota/userquota_001_pos.ksh b/tests/zfs-tests/tests/functional/userquota/userquota_001_pos.ksh
index b134a7677..7a4f8f3ba 100755
--- a/tests/zfs-tests/tests/functional/userquota/userquota_001_pos.ksh
+++ b/tests/zfs-tests/tests/functional/userquota/userquota_001_pos.ksh
@@ -58,7 +58,7 @@ mkmount_writable $QFS
 log_note "Check the userquota@$QUSER1"
 log_must $ZFS set userquota@$QUSER1=$UQUOTA_SIZE $QFS
 log_must user_run $QUSER1 $MKFILE $UQUOTA_SIZE $QFILE
-$SYNC
+sync_pool
 log_mustnot user_run $QUSER1 $MKFILE 1 $OFILE
 cleanup_quota
 
@@ -66,7 +66,7 @@ log_note "Check the groupquota@$QGROUP"
 log_must $ZFS set groupquota@$QGROUP=$GQUOTA_SIZE $QFS
 mkmount_writable $QFS
 log_must user_run $QUSER1 $MKFILE $GQUOTA_SIZE $QFILE
-$SYNC
+sync_pool
 log_mustnot user_run $QUSER1 $MKFILE 1 $OFILE
 
 cleanup_quota
diff --git a/tests/zfs-tests/tests/functional/userquota/userquota_004_pos.ksh b/tests/zfs-tests/tests/functional/userquota/userquota_004_pos.ksh
index 6bdcf1c76..749063261 100755
--- a/tests/zfs-tests/tests/functional/userquota/userquota_004_pos.ksh
+++ b/tests/zfs-tests/tests/functional/userquota/userquota_004_pos.ksh
@@ -50,6 +50,7 @@ log_onexit cleanup
 
 log_assert "Check the basic function of {user|group} used"
 
+sync_pool
 typeset user_used=$(get_value "userused@$QUSER1" $QFS)
 typeset group_used=$(get_value "groupused@$QGROUP" $QFS)
 
@@ -62,7 +63,7 @@ fi
 
 mkmount_writable $QFS
 log_must user_run $QUSER1 $MKFILE 100m $QFILE
-$SYNC
+sync_pool
 
 user_used=$(get_value "userused@$QUSER1" $QFS)
 group_used=$(get_value "groupused@$QGROUP" $QFS)
diff --git a/tests/zfs-tests/tests/functional/userquota/userquota_010_pos.ksh b/tests/zfs-tests/tests/functional/userquota/userquota_010_pos.ksh
index b9260585f..7f7f9672d 100755
--- a/tests/zfs-tests/tests/functional/userquota/userquota_010_pos.ksh
+++ b/tests/zfs-tests/tests/functional/userquota/userquota_010_pos.ksh
@@ -57,7 +57,7 @@ log_must $ZFS set groupquota@$QGROUP=$GQUOTA_SIZE $QFS
 
 mkmount_writable $QFS
 log_must user_run $QUSER1 $MKFILE $UQUOTA_SIZE $QFILE
-$SYNC
+sync_pool
 
 log_must eval "$ZFS get -p userused@$QUSER1 $QFS >/dev/null 2>&1"
 log_must eval "$ZFS get -p groupused@$GROUPUSED $QFS >/dev/null 2>&1"
diff --git a/tests/zfs-tests/tests/functional/userquota/userquota_013_pos.ksh b/tests/zfs-tests/tests/functional/userquota/userquota_013_pos.ksh
new file mode 100644
index 000000000..a84a45524
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/userquota/userquota_013_pos.ksh
@@ -0,0 +1,77 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+# Use is subject to license terms.
+#
+
+#
+# Copyright (c) 2016 by Jinshan Xiong. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/userquota/userquota_common.kshlib
+
+#
+#
+# DESCRIPTION:
+#       Check the basic function of the userobjquota and groupobjquota
+#
+#
+# STRATEGY:
+#       1. Set userobjquota and overwrite the quota size
+#       2. Creating new object should fail with Disc quota exceeded
+#       3. Set groupobjquota and overwrite the quota size
+#       4. Creating new object should fail with Disc quota exceeded
+#
+#
+
+function cleanup
+{
+	log_must $RM -f ${QFILE}_*
+	cleanup_quota
+}
+
+log_onexit cleanup
+
+log_assert "If creating object exceeds {user|group}objquota count, it will fail"
+
+mkmount_writable $QFS
+log_must $ZFS set xattr=sa $QFS
+
+log_note "Check the userobjquota@$QUSER1"
+log_must $ZFS set userobjquota@$QUSER1=100 $QFS
+log_must user_run $QUSER1 $MKFILES ${QFILE}_1 100
+sync_pool
+log_mustnot user_run $QUSER1 $MKFILE 1 $OFILE
+cleanup_quota
+
+log_note "Check the groupobjquota@$QGROUP"
+log_must $ZFS set groupobjquota@$QGROUP=200 $QFS
+mkmount_writable $QFS
+log_must user_run $QUSER1 $MKFILES ${QFILE}_2 100
+sync_pool
+log_mustnot user_run $QUSER2 $MKFILE 1 $OFILE
+
+cleanup
+log_pass "Creating objects exceeds {user|group}objquota count, it as expect"
diff --git a/tests/zfs-tests/tests/functional/userquota/userquota_common.kshlib b/tests/zfs-tests/tests/functional/userquota/userquota_common.kshlib
index 771919602..2b50e293e 100644
--- a/tests/zfs-tests/tests/functional/userquota/userquota_common.kshlib
+++ b/tests/zfs-tests/tests/functional/userquota/userquota_common.kshlib
@@ -38,8 +38,11 @@ function cleanup_quota
 {
 	if datasetexists $QFS; then
 		log_must $ZFS set userquota@$QUSER1=none $QFS
+		log_must $ZFS set userobjquota@$QUSER1=none $QFS
 		log_must $ZFS set userquota@$QUSER2=none $QFS
+		log_must $ZFS set userobjquota@$QUSER2=none $QFS
 		log_must $ZFS set groupquota@$QGROUP=none $QFS
+		log_must $ZFS set groupobjquota@$QGROUP=none $QFS
 		recovery_writable $QFS
 	fi
 
@@ -47,7 +50,7 @@ function cleanup_quota
 	[[ -f $OFILE ]] && log_must $RM -f $OFILE
 	$SYNC
 
-	 return 0
+	return 0
 }
 
 #
diff --git a/tests/zfs-tests/tests/functional/userquota/userspace_002_pos.ksh b/tests/zfs-tests/tests/functional/userquota/userspace_002_pos.ksh
index b29052db6..cb84cf927 100755
--- a/tests/zfs-tests/tests/functional/userquota/userspace_002_pos.ksh
+++ b/tests/zfs-tests/tests/functional/userquota/userspace_002_pos.ksh
@@ -75,7 +75,7 @@ for fs in "$QFS" "$snapfs"; do
 	log_must eval "$ZFS userspace $fs | $GREP $QUSER1 | $GREP 100M"
 
 	log_note "check the user used size in zfs userspace $fs"
-	log_must eval "$ZFS userspace $fs | $GREP $QUSER1 | $GREP 50.0M"
+	log_must eval "$ZFS userspace $fs | $GREP $QUSER1 | $GREP 50\\.\*M"
 done
 
 log_pass "Check the zfs userspace used and quota"
diff --git a/tests/zfs-tests/tests/functional/userquota/userspace_003_pos.ksh b/tests/zfs-tests/tests/functional/userquota/userspace_003_pos.ksh
new file mode 100644
index 000000000..421de6581
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/userquota/userspace_003_pos.ksh
@@ -0,0 +1,116 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+# Use is subject to license terms.
+#
+
+#
+# Copyright (c) 2016 by Jinshan Xiong. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/userquota/userquota_common.kshlib
+
+#
+# DESCRIPTION:
+#       Check the user used object accounting in zfs userspace
+#
+#
+# STRATEGY:
+#       1. create a bunch of files by specific users
+#	2. use zfs userspace to check the used objects
+#	3. change the owner of test files and verify object count
+#	4. delete files and verify object count
+#
+
+function cleanup
+{
+	if datasetexists $snapfs; then
+		log_must $ZFS destroy $snapfs
+	fi
+
+	log_must $RM -f ${QFILE}_*
+	log_must cleanup_quota
+}
+
+function user_object_count
+{
+	typeset fs=$1
+	typeset user=$2
+	typeset cnt=$($ZFS userspace -oname,objused $fs |
+	    $AWK /$user/'{print $2}')
+	echo $cnt
+}
+
+log_onexit cleanup
+
+log_assert "Check the zfs userspace object used"
+
+mkmount_writable $QFS
+log_must $ZFS set xattr=sa $QFS
+
+((user1_cnt = RANDOM % 100 + 1))
+((user2_cnt = RANDOM % 100 + 1))
+
+log_must user_run $QUSER1 $MKFILES ${QFILE}_1 $user1_cnt
+log_must user_run $QUSER2 $MKFILES ${QFILE}_2 $user2_cnt
+sync_pool
+
+typeset snapfs=$QFS@snap
+
+log_must $ZFS snapshot $snapfs
+
+log_must eval "$ZFS userspace $QFS >/dev/null 2>&1"
+log_must eval "$ZFS userspace $snapfs >/dev/null 2>&1"
+
+for fs in "$QFS" "$snapfs"; do
+	log_note "check the user used objects in zfs userspace $fs"
+	[[ $(user_object_count $fs $QUSER1) -eq $user1_cnt ]] ||
+		log_fail "expected $user1_cnt"
+	[[ $(user_object_count $fs $QUSER2) -eq $user2_cnt ]] ||
+		log_fail "expected $user2_cnt"
+done
+
+log_note "change the owner of files"
+log_must $CHOWN $QUSER2 ${QFILE}_1*
+sync_pool
+
+[[ $(user_object_count $QFS $QUSER1) -eq 0 ]] ||
+	log_fail "expected 0 files for $QUSER1"
+
+[[ $(user_object_count $snapfs $QUSER1) -eq $user1_cnt ]] ||
+	log_fail "expected $user_cnt files for $QUSER1 in snapfs"
+
+[[ $(user_object_count $QFS $QUSER2) -eq $((user1_cnt+user2_cnt)) ]] ||
+	log_fail "expected $((user1_cnt+user2_cnt)) files for $QUSER2"
+
+log_note "file removal"
+log_must $RM ${QFILE}_*
+sync_pool
+
+[[ $(user_object_count $QFS $QUSER2) -eq 0 ]] ||
+        log_fail "expected 0 files for $QUSER2"
+
+cleanup
+log_pass "Check the zfs userspace object used"