Simplify threads, mutexs, cvs and rwlocks

* Simplify threads, mutexs, cvs and rwlocks

* Update the zk_thread_create() function to use the same trick
  as Illumos.  Specifically, cast the new pthread_t to a void
  pointer and return that as the kthread_t *.  This avoids the
  issues associated with managing a wrapper structure and is
  safe as long as the callers never attempt to dereference it.

* Update all function prototypes passed to pthread_create() to
  match the expected prototype.  We were getting away this with
  before since the function were explicitly cast.

* Replaced direct zk_thread_create() calls with thread_create()
  for code consistency.  All consumers of libzpool now use the
  proper wrappers.

* The mutex_held() calls were converted to MUTEX_HELD().

* Removed all mutex_owner() calls and retired the interface.
  Instead use MUTEX_HELD() which provides the same information
  and allows the implementation details to be hidden.  In this
  case the use of the pthread_equals() function.

* The kthread_t, kmutex_t, krwlock_t, and krwlock_t types had
  any non essential fields removed.  In the case of kthread_t
  and kcondvar_t they could be directly typedef'd to pthread_t
  and pthread_cond_t respectively.

* Removed all extra ASSERTS from the thread, mutex, rwlock, and
  cv wrapper functions.  In practice, pthreads already provides
  the vast majority of checks as long as we check the return
  code.  Removing this code from our wrappers help readability.

* Added TS_JOINABLE state flag to pass to request a joinable rather
  than detached thread.  This isn't a standard thread_create() state
  but it's the least invasive way to pass this information and is
  only used by ztest.

TEST_ZTEST_TIMEOUT=3600

Chunwei Chen <[email protected]>
Reviewed-by: Tom Caputi <[email protected]>
Signed-off-by: Brian Behlendorf <[email protected]>
Closes #4547 
Closes #5503 
Closes #5523 
Closes #6377 
Closes #6495 

10 files changed, 141 insertions, 354 deletions

diff --git a/cmd/raidz_test/raidz_test.c b/cmd/raidz_test/raidz_test.c
index a2b7e6cbf..a05070399 100644
--- a/cmd/raidz_test/raidz_test.c
+++ b/cmd/raidz_test/raidz_test.c
@@ -702,10 +702,8 @@ run_sweep(void)
 		opts->rto_dsize = size_v[s];
 		opts->rto_v = 0; /* be quiet */
 
-		VERIFY3P(zk_thread_create(NULL, 0,
-		    (thread_func_t)sweep_thread,
-		    (void *) opts, 0, NULL, TS_RUN, 0,
-		    PTHREAD_CREATE_JOINABLE), !=, NULL);
+		VERIFY3P(thread_create(NULL, 0, sweep_thread, (void *) opts,
+		    0, NULL, TS_RUN, defclsyspri), !=, NULL);
 	}
 
 exit:
diff --git a/cmd/ztest/ztest.c b/cmd/ztest/ztest.c
index 788e6aa6a..277782db0 100644
--- a/cmd/ztest/ztest.c
+++ b/cmd/ztest/ztest.c
@@ -2218,7 +2218,7 @@ ztest_lookup(ztest_ds_t *zd, ztest_od_t *od, int count)
 	int error;
 	int i;
 
-	ASSERT(mutex_held(&zd->zd_dirobj_lock));
+	ASSERT(MUTEX_HELD(&zd->zd_dirobj_lock));
 
 	for (i = 0; i < count; i++, od++) {
 		od->od_object = 0;
@@ -2259,7 +2259,7 @@ ztest_create(ztest_ds_t *zd, ztest_od_t *od, int count)
 	int missing = 0;
 	int i;
 
-	ASSERT(mutex_held(&zd->zd_dirobj_lock));
+	ASSERT(MUTEX_HELD(&zd->zd_dirobj_lock));
 
 	for (i = 0; i < count; i++, od++) {
 		if (missing) {
@@ -2305,7 +2305,7 @@ ztest_remove(ztest_ds_t *zd, ztest_od_t *od, int count)
 	int error;
 	int i;
 
-	ASSERT(mutex_held(&zd->zd_dirobj_lock));
+	ASSERT(MUTEX_HELD(&zd->zd_dirobj_lock));
 
 	od += count - 1;
 
@@ -6081,7 +6081,7 @@ ztest_resume(spa_t *spa)
 	(void) zio_resume(spa);
 }
 
-static void *
+static void
 ztest_resume_thread(void *arg)
 {
 	spa_t *spa = arg;
@@ -6105,8 +6105,6 @@ ztest_resume_thread(void *arg)
 	}
 
 	thread_exit();
-
-	return (NULL);
 }
 
 #define	GRACE	300
@@ -6140,7 +6138,7 @@ ztest_execute(int test, ztest_info_t *zi, uint64_t id)
 		    (double)functime / NANOSEC, zi->zi_funcname);
 }
 
-static void *
+static void
 ztest_thread(void *arg)
 {
 	int rand;
@@ -6180,8 +6178,6 @@ ztest_thread(void *arg)
 	}
 
 	thread_exit();
-
-	return (NULL);
 }
 
 static void
@@ -6309,10 +6305,10 @@ ztest_dataset_close(int d)
 static void
 ztest_run(ztest_shared_t *zs)
 {
-	kt_did_t *tid;
 	spa_t *spa;
 	objset_t *os;
 	kthread_t *resume_thread;
+	kthread_t **run_threads;
 	uint64_t object;
 	int error;
 	int t, d;
@@ -6373,9 +6369,8 @@ ztest_run(ztest_shared_t *zs)
 	/*
 	 * Create a thread to periodically resume suspended I/O.
 	 */
-	VERIFY3P((resume_thread = zk_thread_create(NULL, 0,
-	    (thread_func_t)ztest_resume_thread, spa, 0, NULL, TS_RUN, 0,
-	    PTHREAD_CREATE_JOINABLE)), !=, NULL);
+	resume_thread = thread_create(NULL, 0, ztest_resume_thread,
+	    spa, 0, NULL, TS_RUN | TS_JOINABLE, defclsyspri);
 
 #if 0
 	/*
@@ -6409,7 +6404,7 @@ ztest_run(ztest_shared_t *zs)
 	}
 	zs->zs_enospc_count = 0;
 
-	tid = umem_zalloc(ztest_opts.zo_threads * sizeof (kt_did_t),
+	run_threads = umem_zalloc(ztest_opts.zo_threads * sizeof (kthread_t *),
 	    UMEM_NOFAIL);
 
 	if (ztest_opts.zo_verbose >= 4)
@@ -6419,20 +6414,15 @@ ztest_run(ztest_shared_t *zs)
 	 * Kick off all the tests that run in parallel.
 	 */
 	for (t = 0; t < ztest_opts.zo_threads; t++) {
-		kthread_t *thread;
-
-		if (t < ztest_opts.zo_datasets &&
-		    ztest_dataset_open(t) != 0) {
-			umem_free(tid,
-			    ztest_opts.zo_threads * sizeof (kt_did_t));
+		if (t < ztest_opts.zo_datasets && ztest_dataset_open(t) != 0) {
+			umem_free(run_threads, ztest_opts.zo_threads *
+			    sizeof (kthread_t *));
 			return;
 		}
 
-		VERIFY3P(thread = zk_thread_create(NULL, 0,
-		    (thread_func_t)ztest_thread,
-		    (void *)(uintptr_t)t, 0, NULL, TS_RUN, 0,
-		    PTHREAD_CREATE_JOINABLE), !=, NULL);
-		tid[t] = thread->t_tid;
+		run_threads[t] = thread_create(NULL, 0, ztest_thread,
+		    (void *)(uintptr_t)t, 0, NULL, TS_RUN | TS_JOINABLE,
+		    defclsyspri);
 	}
 
 	/*
@@ -6440,7 +6430,7 @@ ztest_run(ztest_shared_t *zs)
 	 * so we don't close datasets while threads are still using them.
 	 */
 	for (t = ztest_opts.zo_threads - 1; t >= 0; t--) {
-		thread_join(tid[t]);
+		VERIFY0(thread_join(run_threads[t]));
 		if (t < ztest_opts.zo_datasets)
 			ztest_dataset_close(t);
 	}
@@ -6450,11 +6440,11 @@ ztest_run(ztest_shared_t *zs)
 	zs->zs_alloc = metaslab_class_get_alloc(spa_normal_class(spa));
 	zs->zs_space = metaslab_class_get_space(spa_normal_class(spa));
 
-	umem_free(tid, ztest_opts.zo_threads * sizeof (kt_did_t));
+	umem_free(run_threads, ztest_opts.zo_threads * sizeof (kthread_t *));
 
 	/* Kill the resume thread */
 	ztest_exiting = B_TRUE;
-	thread_join(resume_thread->t_tid);
+	VERIFY0(thread_join(resume_thread));
 	ztest_resume(spa);
 
 	/*
diff --git a/include/sys/zfs_context.h b/include/sys/zfs_context.h
index 21a9a8481..90e7954a1 100644
--- a/include/sys/zfs_context.h
+++ b/include/sys/zfs_context.h
@@ -207,15 +207,23 @@ extern int aok;
 	(unsigned long)i)
 
 /*
- * Threads.  TS_STACK_MIN is dictated by the minimum allowed pthread stack
- * size.  While TS_STACK_MAX is somewhat arbitrary, it was selected to be
- * large enough for the expected stack depth while small enough to avoid
- * exhausting address space with high thread counts.
+ * Threads.
  */
-#define	TS_MAGIC		0x72f158ab4261e538ull
-#define	TS_RUN			0x00000002
-#define	TS_STACK_MIN		MAX(PTHREAD_STACK_MIN, 32768)
-#define	TS_STACK_MAX		(256 * 1024)
+typedef pthread_t	kthread_t;
+
+#define	TS_RUN		0x00000002
+#define	TS_JOINABLE	0x00000004
+
+#define	curthread	((void *)(uintptr_t)pthread_self())
+#define	kpreempt(x)	yield()
+#define	getcomm()	"unknown"
+
+#define	thread_create(stk, stksize, func, arg, len, pp, state, pri)	\
+	zk_thread_create(func, arg, stksize, state)
+#define	thread_exit()	pthread_exit(NULL)
+#define	thread_join(t)	pthread_join((pthread_t)(t), NULL)
+
+#define	newproc(f, a, cid, pri, ctp, pid)	(ENOSYS)
 
 /* in libzpool, p0 exists only to have its address taken */
 typedef struct proc {
@@ -225,100 +233,55 @@ typedef struct proc {
 extern struct proc p0;
 #define	curproc		(&p0)
 
-typedef void (*thread_func_t)(void *);
-typedef void (*thread_func_arg_t)(void *);
-typedef pthread_t kt_did_t;
-
-#define	kpreempt(x)	((void)0)
-
-typedef struct kthread {
-	kt_did_t	t_tid;
-	thread_func_t	t_func;
-	void *		t_arg;
-	pri_t		t_pri;
-} kthread_t;
+#define	PS_NONE		-1
 
-#define	curthread			zk_thread_current()
-#define	getcomm()			"unknown"
-#define	thread_exit			zk_thread_exit
-#define	thread_create(stk, stksize, func, arg, len, pp, state, pri)	\
-	zk_thread_create(stk, stksize, (thread_func_t)func, arg,	\
-	    len, NULL, state, pri, PTHREAD_CREATE_DETACHED)
-#define	thread_join(t)			zk_thread_join(t)
-#define	newproc(f, a, cid, pri, ctp, pid)	(ENOSYS)
+extern kthread_t *zk_thread_create(void (*func)(void *), void *arg,
+    size_t stksize, int state);
 
-extern kthread_t *zk_thread_current(void);
-extern void zk_thread_exit(void);
-extern kthread_t *zk_thread_create(caddr_t stk, size_t  stksize,
-	thread_func_t func, void *arg, uint64_t len,
-	proc_t *pp, int state, pri_t pri, int detachstate);
-extern void zk_thread_join(kt_did_t tid);
+#define	issig(why)	(FALSE)
+#define	ISSIG(thr, why)	(FALSE)
 
 #define	kpreempt_disable()	((void)0)
 #define	kpreempt_enable()	((void)0)
 
-#define	PS_NONE		-1
-
-#define	issig(why)	(FALSE)
-#define	ISSIG(thr, why)	(FALSE)
-
 /*
  * Mutexes
  */
-#define	MTX_MAGIC	0x9522f51362a6e326ull
-#define	MTX_INIT	((void *)NULL)
-#define	MTX_DEST	((void *)-1UL)
-
 typedef struct kmutex {
-	void		*m_owner;
-	uint64_t	m_magic;
-	pthread_mutex_t	m_lock;
+	pthread_mutex_t		m_lock;
+	pthread_t		m_owner;
 } kmutex_t;
 
-#define	MUTEX_DEFAULT	0
-#define	MUTEX_NOLOCKDEP	MUTEX_DEFAULT
-#define	MUTEX_HELD(m)	((m)->m_owner == curthread)
-#define	MUTEX_NOT_HELD(m) (!MUTEX_HELD(m))
+#define	MUTEX_DEFAULT		0
+#define	MUTEX_NOLOCKDEP		MUTEX_DEFAULT
+#define	MUTEX_HELD(mp)		pthread_equal((mp)->m_owner, pthread_self())
+#define	MUTEX_NOT_HELD(mp)	!MUTEX_HELD(mp)
 
 extern void mutex_init(kmutex_t *mp, char *name, int type, void *cookie);
 extern void mutex_destroy(kmutex_t *mp);
 extern void mutex_enter(kmutex_t *mp);
 extern void mutex_exit(kmutex_t *mp);
 extern int mutex_tryenter(kmutex_t *mp);
-extern void *mutex_owner(kmutex_t *mp);
-extern int mutex_held(kmutex_t *mp);
 
 /*
  * RW locks
  */
-#define	RW_MAGIC	0x4d31fb123648e78aull
-#define	RW_INIT		((void *)NULL)
-#define	RW_DEST		((void *)-1UL)
-
 typedef struct krwlock {
-	void			*rw_owner;
-	void			*rw_wr_owner;
-	uint64_t		rw_magic;
 	pthread_rwlock_t	rw_lock;
+	pthread_t		rw_owner;
 	uint_t			rw_readers;
 } krwlock_t;
 
 typedef int krw_t;
 
-#define	RW_READER	0
-#define	RW_WRITER	1
-#define	RW_DEFAULT	RW_READER
-#define	RW_NOLOCKDEP	RW_READER
+#define	RW_READER		0
+#define	RW_WRITER		1
+#define	RW_DEFAULT		RW_READER
+#define	RW_NOLOCKDEP		RW_READER
 
-#define	RW_READ_HELD(x)		((x)->rw_readers > 0)
-#define	RW_WRITE_HELD(x)	((x)->rw_wr_owner == curthread)
-#define	RW_LOCK_HELD(x)		(RW_READ_HELD(x) || RW_WRITE_HELD(x))
-
-#undef RW_LOCK_HELD
-#define	RW_LOCK_HELD(x)		(RW_READ_HELD(x) || RW_WRITE_HELD(x))
-
-#undef RW_LOCK_HELD
-#define	RW_LOCK_HELD(x)		(RW_READ_HELD(x) || RW_WRITE_HELD(x))
+#define	RW_READ_HELD(rw)	((rw)->rw_readers > 0)
+#define	RW_WRITE_HELD(rw)	pthread_equal((rw)->rw_owner, pthread_self())
+#define	RW_LOCK_HELD(rw)	(RW_READ_HELD(rw) || RW_WRITE_HELD(rw))
 
 extern void rw_init(krwlock_t *rwlp, char *name, int type, void *arg);
 extern void rw_destroy(krwlock_t *rwlp);
@@ -328,6 +291,9 @@ extern int rw_tryupgrade(krwlock_t *rwlp);
 extern void rw_exit(krwlock_t *rwlp);
 #define	rw_downgrade(rwlp) do { } while (0)
 
+/*
+ * Credentials
+ */
 extern uid_t crgetuid(cred_t *cr);
 extern uid_t crgetruid(cred_t *cr);
 extern gid_t crgetgid(cred_t *cr);
@@ -337,14 +303,9 @@ extern gid_t *crgetgroups(cred_t *cr);
 /*
  * Condition variables
  */
-#define	CV_MAGIC	0xd31ea9a83b1b30c4ull
-
-typedef struct kcondvar {
-	uint64_t		cv_magic;
-	pthread_cond_t		cv;
-} kcondvar_t;
+typedef pthread_cond_t		kcondvar_t;
 
-#define	CV_DEFAULT	0
+#define	CV_DEFAULT		0
 #define	CALLOUT_FLAG_ABSOLUTE	0x2
 
 extern void cv_init(kcondvar_t *cv, char *name, int type, void *arg);
@@ -355,6 +316,7 @@ extern clock_t cv_timedwait_hires(kcondvar_t *cvp, kmutex_t *mp, hrtime_t tim,
     hrtime_t res, int flag);
 extern void cv_signal(kcondvar_t *cv);
 extern void cv_broadcast(kcondvar_t *cv);
+
 #define	cv_timedwait_sig(cv, mp, at)		cv_timedwait(cv, mp, at)
 #define	cv_wait_sig(cv, mp)			cv_wait(cv, mp)
 #define	cv_wait_io(cv, mp)			cv_wait(cv, mp)
diff --git a/lib/libzpool/kernel.c b/lib/libzpool/kernel.c
index 3fc7337e8..c4b600d03 100644
--- a/lib/libzpool/kernel.c
+++ b/lib/libzpool/kernel.c
@@ -64,104 +64,29 @@ struct proc p0;
  * =========================================================================
  * threads
  * =========================================================================
+ *
+ * TS_STACK_MIN is dictated by the minimum allowed pthread stack size.  While
+ * TS_STACK_MAX is somewhat arbitrary, it was selected to be large enough for
+ * the expected stack depth while small enough to avoid exhausting address
+ * space with high thread counts.
  */
+#define	TS_STACK_MIN	MAX(PTHREAD_STACK_MIN, 32768)
+#define	TS_STACK_MAX	(256 * 1024)
 
-pthread_cond_t kthread_cond = PTHREAD_COND_INITIALIZER;
-pthread_mutex_t kthread_lock = PTHREAD_MUTEX_INITIALIZER;
-pthread_key_t kthread_key;
-int kthread_nr = 0;
-
-static void
-thread_init(void)
-{
-	kthread_t *kt;
-
-	VERIFY3S(pthread_key_create(&kthread_key, NULL), ==, 0);
-
-	/* Create entry for primary kthread */
-	kt = umem_zalloc(sizeof (kthread_t), UMEM_NOFAIL);
-	kt->t_tid = pthread_self();
-	kt->t_func = NULL;
-
-	VERIFY3S(pthread_setspecific(kthread_key, kt), ==, 0);
-
-	/* Only the main thread should be running at the moment */
-	ASSERT3S(kthread_nr, ==, 0);
-	kthread_nr = 1;
-}
-
-static void
-thread_fini(void)
-{
-	kthread_t *kt = curthread;
-
-	ASSERT(pthread_equal(kt->t_tid, pthread_self()));
-	ASSERT3P(kt->t_func, ==, NULL);
-
-	umem_free(kt, sizeof (kthread_t));
-
-	/* Wait for all threads to exit via thread_exit() */
-	VERIFY3S(pthread_mutex_lock(&kthread_lock), ==, 0);
-
-	kthread_nr--; /* Main thread is exiting */
-
-	while (kthread_nr > 0)
-		VERIFY0(pthread_cond_wait(&kthread_cond, &kthread_lock));
-
-	ASSERT3S(kthread_nr, ==, 0);
-	VERIFY3S(pthread_mutex_unlock(&kthread_lock), ==, 0);
-
-	VERIFY3S(pthread_key_delete(kthread_key), ==, 0);
-}
-
-kthread_t *
-zk_thread_current(void)
-{
-	kthread_t *kt = pthread_getspecific(kthread_key);
-
-	ASSERT3P(kt, !=, NULL);
-
-	return (kt);
-}
-
-void *
-zk_thread_helper(void *arg)
-{
-	kthread_t *kt = (kthread_t *)arg;
-
-	VERIFY3S(pthread_setspecific(kthread_key, kt), ==, 0);
-
-	VERIFY3S(pthread_mutex_lock(&kthread_lock), ==, 0);
-	kthread_nr++;
-	VERIFY3S(pthread_mutex_unlock(&kthread_lock), ==, 0);
-	(void) setpriority(PRIO_PROCESS, 0, kt->t_pri);
-
-	kt->t_tid = pthread_self();
-	((thread_func_arg_t)kt->t_func)(kt->t_arg);
-
-	/* Unreachable, thread must exit with thread_exit() */
-	abort();
-
-	return (NULL);
-}
-
+/*ARGSUSED*/
 kthread_t *
-zk_thread_create(caddr_t stk, size_t stksize, thread_func_t func, void *arg,
-    uint64_t len, proc_t *pp, int state, pri_t pri, int detachstate)
+zk_thread_create(void (*func)(void *), void *arg, size_t stksize, int state)
 {
-	kthread_t *kt;
 	pthread_attr_t attr;
+	pthread_t tid;
 	char *stkstr;
+	int detachstate = PTHREAD_CREATE_DETACHED;
 
-	ASSERT0(state & ~TS_RUN);
-	ASSERT0(len);
+	VERIFY0(pthread_attr_init(&attr));
 
-	kt = umem_zalloc(sizeof (kthread_t), UMEM_NOFAIL);
-	kt->t_func = func;
-	kt->t_arg = arg;
-	kt->t_pri = pri;
+	if (state & TS_JOINABLE)
+		detachstate = PTHREAD_CREATE_JOINABLE;
 
-	VERIFY0(pthread_attr_init(&attr));
 	VERIFY0(pthread_attr_setdetachstate(&attr, detachstate));
 
 	/*
@@ -183,6 +108,7 @@ zk_thread_create(caddr_t stk, size_t stksize, thread_func_t func, void *arg,
 
 	VERIFY3S(stksize, >, 0);
 	stksize = P2ROUNDUP(MAX(stksize, TS_STACK_MIN), PAGESIZE);
+
 	/*
 	 * If this ever fails, it may be because the stack size is not a
 	 * multiple of system page size.
@@ -190,36 +116,10 @@ zk_thread_create(caddr_t stk, size_t stksize, thread_func_t func, void *arg,
 	VERIFY0(pthread_attr_setstacksize(&attr, stksize));
 	VERIFY0(pthread_attr_setguardsize(&attr, PAGESIZE));
 
-	VERIFY0(pthread_create(&kt->t_tid, &attr, &zk_thread_helper, kt));
+	VERIFY0(pthread_create(&tid, &attr, (void *(*)(void *))func, arg));
 	VERIFY0(pthread_attr_destroy(&attr));
 
-	return (kt);
-}
-
-void
-zk_thread_exit(void)
-{
-	kthread_t *kt = curthread;
-
-	ASSERT(pthread_equal(kt->t_tid, pthread_self()));
-
-	umem_free(kt, sizeof (kthread_t));
-
-	VERIFY0(pthread_mutex_lock(&kthread_lock));
-	kthread_nr--;
-	VERIFY0(pthread_mutex_unlock(&kthread_lock));
-
-	VERIFY0(pthread_cond_broadcast(&kthread_cond));
-	pthread_exit((void *)TS_MAGIC);
-}
-
-void
-zk_thread_join(kt_did_t tid)
-{
-	void *ret;
-
-	pthread_join((pthread_t)tid, &ret);
-	VERIFY3P(ret, ==, (void *)TS_MAGIC);
+	return ((void *)(uintptr_t)tid);
 }
 
 /*
@@ -291,46 +191,34 @@ kstat_set_raw_ops(kstat_t *ksp,
 void
 mutex_init(kmutex_t *mp, char *name, int type, void *cookie)
 {
-	ASSERT3S(type, ==, MUTEX_DEFAULT);
-	ASSERT3P(cookie, ==, NULL);
-	mp->m_owner = MTX_INIT;
-	mp->m_magic = MTX_MAGIC;
-	VERIFY3S(pthread_mutex_init(&mp->m_lock, NULL), ==, 0);
+	VERIFY0(pthread_mutex_init(&mp->m_lock, NULL));
+	memset(&mp->m_owner, 0, sizeof (pthread_t));
 }
 
 void
 mutex_destroy(kmutex_t *mp)
 {
-	ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
-	ASSERT3P(mp->m_owner, ==, MTX_INIT);
-	ASSERT0(pthread_mutex_destroy(&(mp)->m_lock));
-	mp->m_owner = MTX_DEST;
-	mp->m_magic = 0;
+	VERIFY0(pthread_mutex_destroy(&mp->m_lock));
 }
 
 void
 mutex_enter(kmutex_t *mp)
 {
-	ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
-	ASSERT3P(mp->m_owner, !=, MTX_DEST);
-	ASSERT3P(mp->m_owner, !=, curthread);
-	VERIFY3S(pthread_mutex_lock(&mp->m_lock), ==, 0);
-	ASSERT3P(mp->m_owner, ==, MTX_INIT);
-	mp->m_owner = curthread;
+	VERIFY0(pthread_mutex_lock(&mp->m_lock));
+	mp->m_owner = pthread_self();
 }
 
 int
 mutex_tryenter(kmutex_t *mp)
 {
-	int err;
-	ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
-	ASSERT3P(mp->m_owner, !=, MTX_DEST);
-	if (0 == (err = pthread_mutex_trylock(&mp->m_lock))) {
-		ASSERT3P(mp->m_owner, ==, MTX_INIT);
-		mp->m_owner = curthread;
+	int error;
+
+	error = pthread_mutex_trylock(&mp->m_lock);
+	if (error == 0) {
+		mp->m_owner = pthread_self();
 		return (1);
 	} else {
-		VERIFY3S(err, ==, EBUSY);
+		VERIFY3S(error, ==, EBUSY);
 		return (0);
 	}
 }
@@ -338,23 +226,8 @@ mutex_tryenter(kmutex_t *mp)
 void
 mutex_exit(kmutex_t *mp)
 {
-	ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
-	ASSERT3P(mutex_owner(mp), ==, curthread);
-	mp->m_owner = MTX_INIT;
-	VERIFY3S(pthread_mutex_unlock(&mp->m_lock), ==, 0);
-}
-
-void *
-mutex_owner(kmutex_t *mp)
-{
-	ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
-	return (mp->m_owner);
-}
-
-int
-mutex_held(kmutex_t *mp)
-{
-	return (mp->m_owner == curthread);
+	memset(&mp->m_owner, 0, sizeof (pthread_t));
+	VERIFY0(pthread_mutex_unlock(&mp->m_lock));
 }
 
 /*
@@ -366,89 +239,60 @@ mutex_held(kmutex_t *mp)
 void
 rw_init(krwlock_t *rwlp, char *name, int type, void *arg)
 {
-	ASSERT3S(type, ==, RW_DEFAULT);
-	ASSERT3P(arg, ==, NULL);
-	VERIFY3S(pthread_rwlock_init(&rwlp->rw_lock, NULL), ==, 0);
-	rwlp->rw_owner = RW_INIT;
-	rwlp->rw_wr_owner = RW_INIT;
+	VERIFY0(pthread_rwlock_init(&rwlp->rw_lock, NULL));
 	rwlp->rw_readers = 0;
-	rwlp->rw_magic = RW_MAGIC;
+	rwlp->rw_owner = 0;
 }
 
 void
 rw_destroy(krwlock_t *rwlp)
 {
-	ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
-	ASSERT(rwlp->rw_readers == 0 && rwlp->rw_wr_owner == RW_INIT);
-	VERIFY3S(pthread_rwlock_destroy(&rwlp->rw_lock), ==, 0);
-	rwlp->rw_magic = 0;
+	VERIFY0(pthread_rwlock_destroy(&rwlp->rw_lock));
 }
 
 void
 rw_enter(krwlock_t *rwlp, krw_t rw)
 {
-	ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
-	ASSERT3P(rwlp->rw_owner, !=, curthread);
-	ASSERT3P(rwlp->rw_wr_owner, !=, curthread);
-
 	if (rw == RW_READER) {
-		VERIFY3S(pthread_rwlock_rdlock(&rwlp->rw_lock), ==, 0);
-		ASSERT3P(rwlp->rw_wr_owner, ==, RW_INIT);
-
+		VERIFY0(pthread_rwlock_rdlock(&rwlp->rw_lock));
 		atomic_inc_uint(&rwlp->rw_readers);
 	} else {
-		VERIFY3S(pthread_rwlock_wrlock(&rwlp->rw_lock), ==, 0);
-		ASSERT3P(rwlp->rw_wr_owner, ==, RW_INIT);
-		ASSERT3U(rwlp->rw_readers, ==, 0);
-
-		rwlp->rw_wr_owner = curthread;
+		VERIFY0(pthread_rwlock_wrlock(&rwlp->rw_lock));
+		rwlp->rw_owner = pthread_self();
 	}
-
-	rwlp->rw_owner = curthread;
 }
 
 void
 rw_exit(krwlock_t *rwlp)
 {
-	ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
-	ASSERT(RW_LOCK_HELD(rwlp));
-
 	if (RW_READ_HELD(rwlp))
 		atomic_dec_uint(&rwlp->rw_readers);
 	else
-		rwlp->rw_wr_owner = RW_INIT;
+		rwlp->rw_owner = 0;
 
-	rwlp->rw_owner = RW_INIT;
-	VERIFY3S(pthread_rwlock_unlock(&rwlp->rw_lock), ==, 0);
+	VERIFY0(pthread_rwlock_unlock(&rwlp->rw_lock));
 }
 
 int
 rw_tryenter(krwlock_t *rwlp, krw_t rw)
 {
-	int rv;
-
-	ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
+	int error;
 
 	if (rw == RW_READER)
-		rv = pthread_rwlock_tryrdlock(&rwlp->rw_lock);
+		error = pthread_rwlock_tryrdlock(&rwlp->rw_lock);
 	else
-		rv = pthread_rwlock_trywrlock(&rwlp->rw_lock);
-
-	if (rv == 0) {
-		ASSERT3P(rwlp->rw_wr_owner, ==, RW_INIT);
+		error = pthread_rwlock_trywrlock(&rwlp->rw_lock);
 
+	if (error == 0) {
 		if (rw == RW_READER)
 			atomic_inc_uint(&rwlp->rw_readers);
-		else {
-			ASSERT3U(rwlp->rw_readers, ==, 0);
-			rwlp->rw_wr_owner = curthread;
-		}
+		else
+			rwlp->rw_owner = pthread_self();
 
-		rwlp->rw_owner = curthread;
 		return (1);
 	}
 
-	VERIFY3S(rv, ==, EBUSY);
+	VERIFY3S(error, ==, EBUSY);
 
 	return (0);
 }
@@ -456,8 +300,6 @@ rw_tryenter(krwlock_t *rwlp, krw_t rw)
 int
 rw_tryupgrade(krwlock_t *rwlp)
 {
-	ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
-
 	return (0);
 }
 
@@ -470,27 +312,21 @@ rw_tryupgrade(krwlock_t *rwlp)
 void
 cv_init(kcondvar_t *cv, char *name, int type, void *arg)
 {
-	ASSERT3S(type, ==, CV_DEFAULT);
-	cv->cv_magic = CV_MAGIC;
-	VERIFY0(pthread_cond_init(&cv->cv, NULL));
+	VERIFY0(pthread_cond_init(cv, NULL));
 }
 
 void
 cv_destroy(kcondvar_t *cv)
 {
-	ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
-	VERIFY0(pthread_cond_destroy(&cv->cv));
-	cv->cv_magic = 0;
+	VERIFY0(pthread_cond_destroy(cv));
 }
 
 void
 cv_wait(kcondvar_t *cv, kmutex_t *mp)
 {
-	ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
-	ASSERT3P(mutex_owner(mp), ==, curthread);
-	mp->m_owner = MTX_INIT;
-	VERIFY0(pthread_cond_wait(&cv->cv, &mp->m_lock));
-	mp->m_owner = curthread;
+	memset(&mp->m_owner, 0, sizeof (pthread_t));
+	VERIFY0(pthread_cond_wait(cv, &mp->m_lock));
+	mp->m_owner = pthread_self();
 }
 
 clock_t
@@ -501,8 +337,6 @@ cv_timedwait(kcondvar_t *cv, kmutex_t *mp, clock_t abstime)
 	timestruc_t ts;
 	clock_t delta;
 
-	ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
-
 	delta = abstime - ddi_get_lbolt();
 	if (delta <= 0)
 		return (-1);
@@ -516,10 +350,9 @@ cv_timedwait(kcondvar_t *cv, kmutex_t *mp, clock_t abstime)
 		ts.tv_nsec -= NANOSEC;
 	}
 
-	ASSERT3P(mutex_owner(mp), ==, curthread);
-	mp->m_owner = MTX_INIT;
-	error = pthread_cond_timedwait(&cv->cv, &mp->m_lock, &ts);
-	mp->m_owner = curthread;
+	memset(&mp->m_owner, 0, sizeof (pthread_t));
+	error = pthread_cond_timedwait(cv, &mp->m_lock, &ts);
+	mp->m_owner = pthread_self();
 
 	if (error == ETIMEDOUT)
 		return (-1);
@@ -548,7 +381,7 @@ cv_timedwait_hires(kcondvar_t *cv, kmutex_t *mp, hrtime_t tim, hrtime_t res,
 	if (delta <= 0)
 		return (-1);
 
-	VERIFY(gettimeofday(&tv, NULL) == 0);
+	VERIFY0(gettimeofday(&tv, NULL));
 
 	ts.tv_sec = tv.tv_sec + delta / NANOSEC;
 	ts.tv_nsec = tv.tv_usec * NSEC_PER_USEC + (delta % NANOSEC);
@@ -557,10 +390,9 @@ cv_timedwait_hires(kcondvar_t *cv, kmutex_t *mp, hrtime_t tim, hrtime_t res,
 		ts.tv_nsec -= NANOSEC;
 	}
 
-	ASSERT(mutex_owner(mp) == curthread);
-	mp->m_owner = MTX_INIT;
-	error = pthread_cond_timedwait(&cv->cv, &mp->m_lock, &ts);
-	mp->m_owner = curthread;
+	memset(&mp->m_owner, 0, sizeof (pthread_t));
+	error = pthread_cond_timedwait(cv, &mp->m_lock, &ts);
+	mp->m_owner = pthread_self();
 
 	if (error == ETIMEDOUT)
 		return (-1);
@@ -573,15 +405,13 @@ cv_timedwait_hires(kcondvar_t *cv, kmutex_t *mp, hrtime_t tim, hrtime_t res,
 void
 cv_signal(kcondvar_t *cv)
 {
-	ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
-	VERIFY0(pthread_cond_signal(&cv->cv));
+	VERIFY0(pthread_cond_signal(cv));
 }
 
 void
 cv_broadcast(kcondvar_t *cv)
 {
-	ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
-	VERIFY0(pthread_cond_broadcast(&cv->cv));
+	VERIFY0(pthread_cond_broadcast(cv));
 }
 
 /*
@@ -1188,7 +1018,6 @@ kernel_init(int mode)
 
 	VERIFY0(uname(&hw_utsname));
 
-	thread_init();
 	system_taskq_init();
 	icp_init();
 
@@ -1207,7 +1036,6 @@ kernel_fini(void)
 
 	icp_fini();
 	system_taskq_fini();
-	thread_fini();
 
 	random_fini();
 }
diff --git a/module/zfs/arc.c b/module/zfs/arc.c
index 481c38189..157a28d4b 100644
--- a/module/zfs/arc.c
+++ b/module/zfs/arc.c
@@ -4214,7 +4214,7 @@ arc_kmem_reap_now(void)
  * using mutex_tryenter() from arc_reclaim_thread().
  */
 static void
-arc_reclaim_thread(void)
+arc_reclaim_thread(void *unused)
 {
 	fstrans_cookie_t	cookie = spl_fstrans_mark();
 	hrtime_t		growtime = 0;
@@ -7515,7 +7515,7 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz)
  * heart of the L2ARC.
  */
 static void
-l2arc_feed_thread(void)
+l2arc_feed_thread(void *unused)
 {
 	callb_cpr_t cpr;
 	l2arc_dev_t *dev;
diff --git a/module/zfs/dbuf.c b/module/zfs/dbuf.c
index dc2c00495..625e06701 100644
--- a/module/zfs/dbuf.c
+++ b/module/zfs/dbuf.c
@@ -531,7 +531,7 @@ dbuf_evict_one(void)
  * out of the cache it is destroyed and becomes eligible for arc eviction.
  */
 static void
-dbuf_evict_thread(void)
+dbuf_evict_thread(void *unused)
 {
 	callb_cpr_t cpr;
 
diff --git a/module/zfs/mmp.c b/module/zfs/mmp.c
index 00478a39f..a4771b677 100644
--- a/module/zfs/mmp.c
+++ b/module/zfs/mmp.c
@@ -123,7 +123,7 @@ uint_t zfs_multihost_import_intervals = MMP_DEFAULT_IMPORT_INTERVALS;
  */
 uint_t zfs_multihost_fail_intervals = MMP_DEFAULT_FAIL_INTERVALS;
 
-static void mmp_thread(spa_t *spa);
+static void mmp_thread(void *arg);
 
 void
 mmp_init(spa_t *spa)
@@ -364,8 +364,9 @@ mmp_write_uberblock(spa_t *spa)
 }
 
 static void
-mmp_thread(spa_t *spa)
+mmp_thread(void *arg)
 {
+	spa_t *spa = (spa_t *)arg;
 	mmp_thread_t *mmp = &spa->spa_mmp;
 	boolean_t last_spa_suspended = spa_suspended(spa);
 	boolean_t last_spa_multihost = spa_multihost(spa);
diff --git a/module/zfs/spa.c b/module/zfs/spa.c
index f1f1444f1..cb86c6200 100644
--- a/module/zfs/spa.c
+++ b/module/zfs/spa.c
@@ -1028,6 +1028,11 @@ spa_create_zio_taskqs(spa_t *spa)
 	}
 }
 
+/*
+ * Disabled until spa_thread() can be adapted for Linux.
+ */
+#undef HAVE_SPA_THREAD
+
 #if defined(_KERNEL) && defined(HAVE_SPA_THREAD)
 static void
 spa_thread(void *arg)
@@ -3415,7 +3420,7 @@ spa_open_common(const char *pool, spa_t **spapp, void *tag, nvlist_t *nvpolicy,
 	 * up calling spa_open() again.  The real fix is to figure out how to
 	 * avoid dsl_dir_open() calling this in the first place.
 	 */
-	if (mutex_owner(&spa_namespace_lock) != curthread) {
+	if (MUTEX_NOT_HELD(&spa_namespace_lock)) {
 		mutex_enter(&spa_namespace_lock);
 		locked = B_TRUE;
 	}
@@ -6068,8 +6073,9 @@ spa_async_autoexpand(spa_t *spa, vdev_t *vd)
 }
 
 static void
-spa_async_thread(spa_t *spa)
+spa_async_thread(void *arg)
 {
+	spa_t *spa = (spa_t *)arg;
 	int tasks, i;
 
 	ASSERT(spa->spa_sync_on);
diff --git a/module/zfs/txg.c b/module/zfs/txg.c
index 65bd7f93a..8b1ec9c05 100644
--- a/module/zfs/txg.c
+++ b/module/zfs/txg.c
@@ -108,8 +108,8 @@
  * now transition to the syncing state.
  */
 
-static void txg_sync_thread(dsl_pool_t *dp);
-static void txg_quiesce_thread(dsl_pool_t *dp);
+static void txg_sync_thread(void *dp);
+static void txg_quiesce_thread(void *dp);
 
 int zfs_txg_timeout = 5;	/* max seconds worth of delta per txg */
 
@@ -477,8 +477,9 @@ txg_wait_callbacks(dsl_pool_t *dp)
 }
 
 static void
-txg_sync_thread(dsl_pool_t *dp)
+txg_sync_thread(void *arg)
 {
+	dsl_pool_t *dp = (dsl_pool_t *)arg;
 	spa_t *spa = dp->dp_spa;
 	tx_state_t *tx = &dp->dp_tx;
 	callb_cpr_t cpr;
@@ -561,8 +562,9 @@ txg_sync_thread(dsl_pool_t *dp)
 }
 
 static void
-txg_quiesce_thread(dsl_pool_t *dp)
+txg_quiesce_thread(void *arg)
 {
+	dsl_pool_t *dp = (dsl_pool_t *)arg;
 	tx_state_t *tx = &dp->dp_tx;
 	callb_cpr_t cpr;
 
diff --git a/module/zfs/zfs_sa.c b/module/zfs/zfs_sa.c
index 13e99c058..08e881cc3 100644
--- a/module/zfs/zfs_sa.c
+++ b/module/zfs/zfs_sa.c
@@ -300,7 +300,7 @@ zfs_sa_upgrade(sa_handle_t *hdl, dmu_tx_t *tx)
 	 * Otherwise, we know we are doing the
 	 * sa_update() that caused us to enter this function.
 	 */
-	if (mutex_owner(&zp->z_lock) != curthread) {
+	if (MUTEX_NOT_HELD(&zp->z_lock)) {
 		if (mutex_tryenter(&zp->z_lock) == 0)
 			return;
 		else