Fix Solaris thread dependency by using pthreads

This is a portability change which removes the dependence of the Solaris
thread library.  All locations where Solaris thread API was used before
have been replaced with equivilant Solaris kernel style thread calls.
In user space the kernel style threading API is implemented in term of
the portable pthreads library.  This includes all threads, mutexs,
condition variables, reader/writer locks, and taskqs.

Signed-off-by: Brian Behlendorf <[email protected]>

4 files changed, 357 insertions, 135 deletions

diff --git a/lib/libuutil/uu_misc.c b/lib/libuutil/uu_misc.c
index 578bf3294..1b843effe 100644
--- a/lib/libuutil/uu_misc.c
+++ b/lib/libuutil/uu_misc.c
@@ -34,7 +34,6 @@
 #include <stdlib.h>
 #include <string.h>
 #include <sys/debug.h>
-#include <thread.h>
 #include <unistd.h>
 #include <ctype.h>
 
@@ -68,11 +67,12 @@ static va_list		uu_panic_args;
 static pthread_t	uu_panic_thread;
 
 static uint32_t		_uu_main_error;
+static __thread int	_uu_main_thread = 0;
 
 void
 uu_set_error(uint_t code)
 {
-	if (thr_main() != 0) {
+	if (_uu_main_thread) {
 		_uu_main_error = code;
 		return;
 	}
@@ -101,7 +101,7 @@ uu_set_error(uint_t code)
 uint32_t
 uu_error(void)
 {
-	if (thr_main() != 0)
+	if (_uu_main_thread)
 		return (_uu_main_error);
 
 	if (uu_error_key_setup < 0)	/* can't happen? */
@@ -259,6 +259,7 @@ uu_init(void) __attribute__((constructor));
 static void
 uu_init(void)
 {
+	_uu_main_thread = 1;
 	(void) pthread_atfork(uu_lockup, uu_release, uu_release_child);
 }
 
diff --git a/lib/libzpool/include/sys/zfs_context.h b/lib/libzpool/include/sys/zfs_context.h
index fc543559b..55b117c21 100644
--- a/lib/libzpool/include/sys/zfs_context.h
+++ b/lib/libzpool/include/sys/zfs_context.h
@@ -49,8 +49,7 @@ extern "C" {
 #include <errno.h>
 #include <string.h>
 #include <strings.h>
-#include <synch.h>
-#include <thread.h>
+#include <pthread.h>
 #include <assert.h>
 #include <alloca.h>
 #include <umem.h>
@@ -90,6 +89,8 @@ extern "C" {
 #define	CE_PANIC	3	/* panic		*/
 #define	CE_IGNORE	4	/* print nothing	*/
 
+extern int aok;
+
 /*
  * ZFS debugging
  */
@@ -195,27 +196,55 @@ _NOTE(CONSTCOND) } while (0)
 /*
  * Threads
  */
-#define	curthread	((void *)(uintptr_t)thr_self())
-
-typedef struct kthread kthread_t;
-
-#define	thread_create(stk, stksize, func, arg, len, pp, state, pri)	\
-	zk_thread_create(func, arg)
-#define	thread_exit() thr_exit(NULL)
-#define	thread_join(t)	panic("libzpool cannot join threads")
+#define	TS_MAGIC		0x72f158ab4261e538ull
+#define	TS_RUN			0x00000002
+#ifdef __linux__
+#define	STACK_SIZE		8192	/* Linux x86 and amd64 */
+#else
+#define	STACK_SIZE		24576	/* Solaris */
+#endif
 
-#define	newproc(f, a, cid, pri, ctp, pid)	(ENOSYS)
+#ifdef NPTL_GUARD_WITHIN_STACK
+#define	EXTRA_GUARD_BYTES	PAGESIZE
+#else
+#define	EXTRA_GUARD_BYTES	0
+#endif
 
 /* in libzpool, p0 exists only to have its address taken */
-struct proc {
+typedef struct proc {
 	uintptr_t	this_is_never_used_dont_dereference_it;
-};
+} proc_t;
 
 extern struct proc p0;
 
-#define	PS_NONE		-1
+typedef void (*thread_func_t)(void *);
+typedef void (*thread_func_arg_t)(void *);
+typedef pthread_t kt_did_t;
 
-extern kthread_t *zk_thread_create(void (*func)(), void *arg);
+typedef struct kthread {
+	kt_did_t	t_tid;
+	thread_func_t	t_func;
+	void *		t_arg;
+} kthread_t;
+
+#define	tsd_get(key)			pthread_getspecific(key)
+#define	tsd_set(key, val)		pthread_setspecific(key, val)
+#define	curthread			zk_thread_current()
+#define	thread_exit			zk_thread_exit
+#define	thread_create(stk, stksize, func, arg, len, pp, state, pri)	\
+	zk_thread_create(stk, stksize, (thread_func_t)func, arg,	\
+			 len, NULL, state, pri)
+#define	thread_join(t)			zk_thread_join(t)
+#define	newproc(f,a,cid,pri,ctp,pid)	(ENOSYS)
+
+extern kthread_t *zk_thread_current(void);
+extern void zk_thread_exit(void);
+extern kthread_t *zk_thread_create(caddr_t stk, size_t  stksize,
+	thread_func_t func, void *arg, size_t len,
+	proc_t *pp, int state, pri_t pri);
+extern void zk_thread_join(kt_did_t tid);
+
+#define	PS_NONE		-1
 
 #define	issig(why)	(FALSE)
 #define	ISSIG(thr, why)	(FALSE)
@@ -223,55 +252,52 @@ extern kthread_t *zk_thread_create(void (*func)(), void *arg);
 /*
  * Mutexes
  */
+#define	MTX_MAGIC	0x9522f51362a6e326ull
+#define	MTX_INIT	((void *)NULL)
+#define	MTX_DEST	((void *)-1UL)
+
 typedef struct kmutex {
 	void		*m_owner;
-	boolean_t	initialized;
-	mutex_t		m_lock;
+	uint64_t	m_magic;
+	pthread_mutex_t	m_lock;
 } kmutex_t;
 
-#define	MUTEX_DEFAULT	USYNC_THREAD
-#undef	MUTEX_HELD
-#undef	MUTEX_NOT_HELD
-#define	MUTEX_HELD(m) _mutex_held(&(m)->m_lock)
+#define	MUTEX_DEFAULT	0
+#define MUTEX_HELD(m)	((m)->m_owner == curthread)
 #define	MUTEX_NOT_HELD(m) (!MUTEX_HELD(m))
 
-/*
- * Argh -- we have to get cheesy here because the kernel and userland
- * have different signatures for the same routine.
- */
-extern int _mutex_init(mutex_t *mp, int type, void *arg);
-extern int _mutex_destroy(mutex_t *mp);
-
-#define	mutex_init(mp, b, c, d)		zmutex_init((kmutex_t *)(mp))
-#define	mutex_destroy(mp)		zmutex_destroy((kmutex_t *)(mp))
-
-extern void zmutex_init(kmutex_t *mp);
-extern void zmutex_destroy(kmutex_t *mp);
+extern void mutex_init(kmutex_t *mp, char *name, int type, void *cookie);
+extern void mutex_destroy(kmutex_t *mp);
 extern void mutex_enter(kmutex_t *mp);
 extern void mutex_exit(kmutex_t *mp);
 extern int mutex_tryenter(kmutex_t *mp);
 extern void *mutex_owner(kmutex_t *mp);
+extern int mutex_held(kmutex_t *mp);
 
 /*
  * RW locks
  */
+#define	RW_MAGIC	0x4d31fb123648e78aull
+#define	RW_INIT		((void *)NULL)
+#define	RW_DEST		((void *)-1UL)
+
 typedef struct krwlock {
-	void		*rw_owner;
-	boolean_t	initialized;
-	rwlock_t	rw_lock;
+	void			*rw_owner;
+	void			*rw_wr_owner;
+	uint64_t		rw_magic;
+	pthread_rwlock_t	rw_lock;
+	uint_t			rw_readers;
 } krwlock_t;
 
 typedef int krw_t;
 
 #define	RW_READER	0
 #define	RW_WRITER	1
-#define	RW_DEFAULT	USYNC_THREAD
+#define RW_DEFAULT	RW_READER
 
-#undef RW_READ_HELD
-#define	RW_READ_HELD(x)		_rw_read_held(&(x)->rw_lock)
-
-#undef RW_WRITE_HELD
-#define	RW_WRITE_HELD(x)	_rw_write_held(&(x)->rw_lock)
+#define	RW_READ_HELD(x)		((x)->rw_readers > 0)
+#define	RW_WRITE_HELD(x)	((x)->rw_wr_owner == curthread)
+#define	RW_LOCK_HELD(x)		(RW_READ_HELD(x) || RW_WRITE_HELD(x))
 
 extern void rw_init(krwlock_t *rwlp, char *name, int type, void *arg);
 extern void rw_destroy(krwlock_t *rwlp);
@@ -289,9 +315,14 @@ extern gid_t *crgetgroups(cred_t *cr);
 /*
  * Condition variables
  */
-typedef cond_t kcondvar_t;
+#define CV_MAGIC	0xd31ea9a83b1b30c4ull
+
+typedef struct kcondvar {
+	uint64_t		cv_magic;
+	pthread_cond_t		cv;
+} kcondvar_t;
 
-#define	CV_DEFAULT	USYNC_THREAD
+#define	CV_DEFAULT	0
 
 extern void cv_init(kcondvar_t *cv, char *name, int type, void *arg);
 extern void cv_destroy(kcondvar_t *cv);
@@ -367,7 +398,7 @@ extern taskq_t	*taskq_create(const char *, int, pri_t, int, int, uint_t);
 extern taskqid_t taskq_dispatch(taskq_t *, task_func_t, void *, uint_t);
 extern void	taskq_destroy(taskq_t *);
 extern void	taskq_wait(taskq_t *);
-extern int	taskq_member(taskq_t *, void *);
+extern int	taskq_member(taskq_t *, kthread_t *);
 extern void	system_taskq_init(void);
 extern void	system_taskq_fini(void);
 
@@ -496,7 +527,7 @@ extern void delay(clock_t ticks);
 #define	minclsyspri	60
 #define	maxclsyspri	99
 
-#define	CPU_SEQID	(thr_self() & (max_ncpus - 1))
+#define	CPU_SEQID	(pthread_self() & (max_ncpus - 1))
 
 #define	kcred		NULL
 #define	CRED()		NULL
diff --git a/lib/libzpool/kernel.c b/lib/libzpool/kernel.c
index b64f03da4..4bd08cdd4 100644
--- a/lib/libzpool/kernel.c
+++ b/lib/libzpool/kernel.c
@@ -29,6 +29,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <zlib.h>
+#include <sys/signal.h>
 #include <sys/spa.h>
 #include <sys/stat.h>
 #include <sys/processor.h>
@@ -57,16 +58,156 @@ struct proc p0;
  * threads
  * =========================================================================
  */
-/*ARGSUSED*/
+
+pthread_cond_t kthread_cond = PTHREAD_COND_INITIALIZER;
+pthread_mutex_t kthread_lock = PTHREAD_MUTEX_INITIALIZER;
+pthread_key_t kthread_key;
+int kthread_nr = 0;
+
+static void
+thread_init(void)
+{
+	kthread_t *kt;
+
+	VERIFY3S(pthread_key_create(&kthread_key, NULL), ==, 0);
+
+	/* Create entry for primary kthread */
+	kt = umem_zalloc(sizeof(kthread_t), UMEM_NOFAIL);
+	kt->t_tid = pthread_self();
+	kt->t_func = NULL;
+
+	VERIFY3S(pthread_setspecific(kthread_key, kt), ==, 0);
+
+	/* Only the main thread should be running at the moment */
+	ASSERT3S(kthread_nr, ==, 0);
+	kthread_nr = 1;
+}
+
+static void
+thread_fini(void)
+{
+	kthread_t *kt = curthread;
+
+	ASSERT(pthread_equal(kt->t_tid, pthread_self()));
+	ASSERT3P(kt->t_func, ==, NULL);
+
+	umem_free(kt, sizeof(kthread_t));
+
+	/* Wait for all threads to exit via thread_exit() */
+	VERIFY3S(pthread_mutex_lock(&kthread_lock), ==, 0);
+
+	kthread_nr--; /* Main thread is exiting */
+
+	while (kthread_nr > 0)
+		VERIFY3S(pthread_cond_wait(&kthread_cond, &kthread_lock), ==,
+		    0);
+
+	ASSERT3S(kthread_nr, ==, 0);
+	VERIFY3S(pthread_mutex_unlock(&kthread_lock), ==, 0);
+
+	VERIFY3S(pthread_key_delete(kthread_key), ==, 0);
+}
+
 kthread_t *
-zk_thread_create(void (*func)(), void *arg)
+zk_thread_current(void)
+{
+	kthread_t *kt = pthread_getspecific(kthread_key);
+
+	ASSERT3P(kt, !=, NULL);
+
+	return kt;
+}
+
+void *
+zk_thread_helper(void *arg)
 {
-	thread_t tid;
+	kthread_t *kt = (kthread_t *) arg;
+
+	VERIFY3S(pthread_setspecific(kthread_key, kt), ==, 0);
 
-	VERIFY(thr_create(0, 0, (void *(*)(void *))func, arg, THR_DETACHED,
-	    &tid) == 0);
+	VERIFY3S(pthread_mutex_lock(&kthread_lock), ==, 0);
+	kthread_nr++;
+	VERIFY3S(pthread_mutex_unlock(&kthread_lock), ==, 0);
 
-	return ((void *)(uintptr_t)tid);
+	kt->t_tid = pthread_self();
+	((thread_func_arg_t) kt->t_func)(kt->t_arg);
+
+	/* Unreachable, thread must exit with thread_exit() */
+	abort();
+
+	return NULL;
+}
+
+kthread_t *
+zk_thread_create(caddr_t stk, size_t stksize, thread_func_t func, void *arg,
+	      size_t len, proc_t *pp, int state, pri_t pri)
+{
+	kthread_t *kt;
+	pthread_attr_t attr;
+	size_t stack;
+
+	ASSERT3S(state & ~TS_RUN, ==, 0);
+
+	kt = umem_zalloc(sizeof(kthread_t), UMEM_NOFAIL);
+	kt->t_func = func;
+	kt->t_arg = arg;
+
+	/*
+	 * The Solaris kernel stack size is 24k for x86/x86_64.
+	 * The Linux kernel stack size is 8k for x86/x86_64.
+	 *
+	 * We reduce the default stack size in userspace, to ensure
+	 * we observe stack overruns in user space as well as in
+	 * kernel space.  PTHREAD_STACK_MIN is the minimum stack
+	 * required for a NULL procedure in user space and is added
+	 * in to the stack requirements.
+	 *
+	 * Some buggy NPTL threading implementations include the
+	 * guard area within the stack size allocations.  In
+	 * this case we allocate an extra page to account for the
+	 * guard area since we only have two pages of usable stack
+	 * on Linux.
+	 */
+
+	stack = PTHREAD_STACK_MIN + MAX(stksize, STACK_SIZE) +
+			EXTRA_GUARD_BYTES;
+
+	VERIFY3S(pthread_attr_init(&attr), ==, 0);
+	VERIFY3S(pthread_attr_setstacksize(&attr, stack), ==, 0);
+	VERIFY3S(pthread_attr_setguardsize(&attr, PAGESIZE), ==, 0);
+
+	VERIFY3S(pthread_create(&kt->t_tid, &attr, &zk_thread_helper, kt),
+	    ==, 0);
+
+	VERIFY3S(pthread_attr_destroy(&attr), ==, 0);
+
+	return kt;
+}
+
+void
+zk_thread_exit(void)
+{
+	kthread_t *kt = curthread;
+
+	ASSERT(pthread_equal(kt->t_tid, pthread_self()));
+
+	umem_free(kt, sizeof(kthread_t));
+
+	pthread_mutex_lock(&kthread_lock);
+	kthread_nr--;
+	pthread_mutex_unlock(&kthread_lock);
+
+	pthread_cond_broadcast(&kthread_cond);
+	pthread_exit((void *)TS_MAGIC);
+}
+
+void
+zk_thread_join(kt_did_t tid)
+{
+	void *ret;
+
+	pthread_join((pthread_t)tid, &ret);
+	VERIFY3P(ret, ==, (void *)TS_MAGIC);
 }
 
 /*
@@ -97,42 +238,45 @@ kstat_delete(kstat_t *ksp)
  * mutexes
  * =========================================================================
  */
+
 void
-zmutex_init(kmutex_t *mp)
+mutex_init(kmutex_t *mp, char *name, int type, void *cookie)
 {
-	mp->m_owner = NULL;
-	mp->initialized = B_TRUE;
-	(void) _mutex_init(&mp->m_lock, USYNC_THREAD, NULL);
+	ASSERT3S(type, ==, MUTEX_DEFAULT);
+	ASSERT3P(cookie, ==, NULL);
+	mp->m_owner = MTX_INIT;
+	mp->m_magic = MTX_MAGIC;
+	VERIFY3S(pthread_mutex_init(&mp->m_lock, NULL), ==, 0);
 }
 
 void
-zmutex_destroy(kmutex_t *mp)
+mutex_destroy(kmutex_t *mp)
 {
-	ASSERT(mp->initialized == B_TRUE);
-	ASSERT(mp->m_owner == NULL);
-	(void) _mutex_destroy(&(mp)->m_lock);
-	mp->m_owner = (void *)-1UL;
-	mp->initialized = B_FALSE;
+	ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
+	ASSERT3P(mp->m_owner, ==, MTX_INIT);
+	VERIFY3S(pthread_mutex_destroy(&(mp)->m_lock), ==, 0);
+	mp->m_owner = MTX_DEST;
+	mp->m_magic = 0;
 }
 
 void
 mutex_enter(kmutex_t *mp)
 {
-	ASSERT(mp->initialized == B_TRUE);
-	ASSERT(mp->m_owner != (void *)-1UL);
-	ASSERT(mp->m_owner != curthread);
-	VERIFY(mutex_lock(&mp->m_lock) == 0);
-	ASSERT(mp->m_owner == NULL);
+	ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
+	ASSERT3P(mp->m_owner, !=, MTX_DEST);
+	ASSERT3P(mp->m_owner, !=, curthread);
+	VERIFY3S(pthread_mutex_lock(&mp->m_lock), ==, 0);
+	ASSERT3P(mp->m_owner, ==, MTX_INIT);
 	mp->m_owner = curthread;
 }
 
 int
 mutex_tryenter(kmutex_t *mp)
 {
-	ASSERT(mp->initialized == B_TRUE);
-	ASSERT(mp->m_owner != (void *)-1UL);
-	if (0 == mutex_trylock(&mp->m_lock)) {
-		ASSERT(mp->m_owner == NULL);
+	ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
+	ASSERT3P(mp->m_owner, !=, MTX_DEST);
+	if (0 == pthread_mutex_trylock(&mp->m_lock)) {
+		ASSERT3P(mp->m_owner, ==, MTX_INIT);
 		mp->m_owner = curthread;
 		return (1);
 	} else {
@@ -143,53 +287,71 @@ mutex_tryenter(kmutex_t *mp)
 void
 mutex_exit(kmutex_t *mp)
 {
-	ASSERT(mp->initialized == B_TRUE);
-	ASSERT(mutex_owner(mp) == curthread);
-	mp->m_owner = NULL;
-	VERIFY(mutex_unlock(&mp->m_lock) == 0);
+	ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
+	ASSERT3P(mutex_owner(mp), ==, curthread);
+	mp->m_owner = MTX_INIT;
+	VERIFY3S(pthread_mutex_unlock(&mp->m_lock), ==, 0);
 }
 
 void *
 mutex_owner(kmutex_t *mp)
 {
-	ASSERT(mp->initialized == B_TRUE);
+	ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
 	return (mp->m_owner);
 }
 
+int
+mutex_held(kmutex_t *mp)
+{
+	return (mp->m_owner == curthread);
+}
+
 /*
  * =========================================================================
  * rwlocks
  * =========================================================================
  */
-/*ARGSUSED*/
+
 void
 rw_init(krwlock_t *rwlp, char *name, int type, void *arg)
 {
-	rwlock_init(&rwlp->rw_lock, USYNC_THREAD, NULL);
-	rwlp->rw_owner = NULL;
-	rwlp->initialized = B_TRUE;
+	ASSERT3S(type, ==, RW_DEFAULT);
+	ASSERT3P(arg, ==, NULL);
+	VERIFY3S(pthread_rwlock_init(&rwlp->rw_lock, NULL), ==, 0);
+	rwlp->rw_owner = RW_INIT;
+	rwlp->rw_wr_owner = RW_INIT;
+	rwlp->rw_readers = 0;
+	rwlp->rw_magic = RW_MAGIC;
 }
 
 void
 rw_destroy(krwlock_t *rwlp)
 {
-	rwlock_destroy(&rwlp->rw_lock);
-	rwlp->rw_owner = (void *)-1UL;
-	rwlp->initialized = B_FALSE;
+	ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
+
+	VERIFY3S(pthread_rwlock_destroy(&rwlp->rw_lock), ==, 0);
+	rwlp->rw_magic = 0;
 }
 
 void
 rw_enter(krwlock_t *rwlp, krw_t rw)
 {
-	ASSERT(!RW_LOCK_HELD(rwlp));
-	ASSERT(rwlp->initialized == B_TRUE);
-	ASSERT(rwlp->rw_owner != (void *)-1UL);
-	ASSERT(rwlp->rw_owner != curthread);
+	ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
+	ASSERT3P(rwlp->rw_owner, !=, curthread);
+	ASSERT3P(rwlp->rw_wr_owner, !=, curthread);
 
-	if (rw == RW_READER)
-		VERIFY(rw_rdlock(&rwlp->rw_lock) == 0);
-	else
-		VERIFY(rw_wrlock(&rwlp->rw_lock) == 0);
+	if (rw == RW_READER) {
+		VERIFY3S(pthread_rwlock_rdlock(&rwlp->rw_lock), ==, 0);
+		ASSERT3P(rwlp->rw_wr_owner, ==, RW_INIT);
+
+		atomic_inc_uint(&rwlp->rw_readers);
+	} else {
+		VERIFY3S(pthread_rwlock_wrlock(&rwlp->rw_lock), ==, 0);
+		ASSERT3P(rwlp->rw_wr_owner, ==, RW_INIT);
+		ASSERT3U(rwlp->rw_readers, ==, 0);
+
+		rwlp->rw_wr_owner = curthread;
+	}
 
 	rwlp->rw_owner = curthread;
 }
@@ -197,11 +359,16 @@ rw_enter(krwlock_t *rwlp, krw_t rw)
 void
 rw_exit(krwlock_t *rwlp)
 {
-	ASSERT(rwlp->initialized == B_TRUE);
-	ASSERT(rwlp->rw_owner != (void *)-1UL);
+	ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
+	ASSERT(RW_LOCK_HELD(rwlp));
+
+	if (RW_READ_HELD(rwlp))
+		atomic_dec_uint(&rwlp->rw_readers);
+	else
+		rwlp->rw_wr_owner = RW_INIT;
 
-	rwlp->rw_owner = NULL;
-	VERIFY(rw_unlock(&rwlp->rw_lock) == 0);
+	rwlp->rw_owner = RW_INIT;
+	VERIFY3S(pthread_rwlock_unlock(&rwlp->rw_lock), ==, 0);
 }
 
 int
@@ -209,28 +376,36 @@ rw_tryenter(krwlock_t *rwlp, krw_t rw)
 {
 	int rv;
 
-	ASSERT(rwlp->initialized == B_TRUE);
-	ASSERT(rwlp->rw_owner != (void *)-1UL);
+	ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
 
 	if (rw == RW_READER)
-		rv = rw_tryrdlock(&rwlp->rw_lock);
+		rv = pthread_rwlock_tryrdlock(&rwlp->rw_lock);
 	else
-		rv = rw_trywrlock(&rwlp->rw_lock);
+		rv = pthread_rwlock_trywrlock(&rwlp->rw_lock);
 
 	if (rv == 0) {
+		ASSERT3P(rwlp->rw_wr_owner, ==, RW_INIT);
+
+		if (rw == RW_READER)
+			atomic_inc_uint(&rwlp->rw_readers);
+		else {
+			ASSERT3U(rwlp->rw_readers, ==, 0);
+			rwlp->rw_wr_owner = curthread;
+		}
+
 		rwlp->rw_owner = curthread;
 		return (1);
 	}
 
+	VERIFY3S(rv, ==, EBUSY);
+
 	return (0);
 }
 
-/*ARGSUSED*/
 int
 rw_tryupgrade(krwlock_t *rwlp)
 {
-	ASSERT(rwlp->initialized == B_TRUE);
-	ASSERT(rwlp->rw_owner != (void *)-1UL);
+	ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
 
 	return (0);
 }
@@ -240,26 +415,32 @@ rw_tryupgrade(krwlock_t *rwlp)
  * condition variables
  * =========================================================================
  */
-/*ARGSUSED*/
+
 void
 cv_init(kcondvar_t *cv, char *name, int type, void *arg)
 {
-	VERIFY(cond_init(cv, type, NULL) == 0);
+	ASSERT3S(type, ==, CV_DEFAULT);
+	cv->cv_magic = CV_MAGIC;
+	VERIFY3S(pthread_cond_init(&cv->cv, NULL), ==, 0);
 }
 
 void
 cv_destroy(kcondvar_t *cv)
 {
-	VERIFY(cond_destroy(cv) == 0);
+	ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
+	VERIFY3S(pthread_cond_destroy(&cv->cv), ==, 0);
+	cv->cv_magic = 0;
 }
 
 void
 cv_wait(kcondvar_t *cv, kmutex_t *mp)
 {
-	ASSERT(mutex_owner(mp) == curthread);
-	mp->m_owner = NULL;
-	int ret = cond_wait(cv, &mp->m_lock);
-	VERIFY(ret == 0 || ret == EINTR);
+	ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
+	ASSERT3P(mutex_owner(mp), ==, curthread);
+	mp->m_owner = MTX_INIT;
+	int ret = pthread_cond_wait(&cv->cv, &mp->m_lock);
+	if (ret != 0)
+		VERIFY3S(ret, ==, EINTR);
 	mp->m_owner = curthread;
 }
 
@@ -267,29 +448,38 @@ clock_t
 cv_timedwait(kcondvar_t *cv, kmutex_t *mp, clock_t abstime)
 {
 	int error;
+	struct timeval tv;
 	timestruc_t ts;
 	clock_t delta;
 
+	ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
+
 top:
 	delta = abstime - ddi_get_lbolt();
 	if (delta <= 0)
 		return (-1);
 
-	ts.tv_sec = delta / hz;
-	ts.tv_nsec = (delta % hz) * (NANOSEC / hz);
+	VERIFY(gettimeofday(&tv, NULL) == 0);
+
+	ts.tv_sec = tv.tv_sec + delta / hz;
+	ts.tv_nsec = tv.tv_usec * 1000 + (delta % hz) * (NANOSEC / hz);
+	if (ts.tv_nsec >= NANOSEC) {
+		ts.tv_sec++;
+		ts.tv_nsec -= NANOSEC;
+	}
 
-	ASSERT(mutex_owner(mp) == curthread);
-	mp->m_owner = NULL;
-	error = cond_reltimedwait(cv, &mp->m_lock, &ts);
+	ASSERT3P(mutex_owner(mp), ==, curthread);
+	mp->m_owner = MTX_INIT;
+	error = pthread_cond_timedwait(&cv->cv, &mp->m_lock, &ts);
 	mp->m_owner = curthread;
 
-	if (error == ETIME)
+	if (error == ETIMEDOUT)
 		return (-1);
 
 	if (error == EINTR)
 		goto top;
 
-	ASSERT(error == 0);
+	VERIFY3S(error, ==, 0);
 
 	return (1);
 }
@@ -297,13 +487,15 @@ top:
 void
 cv_signal(kcondvar_t *cv)
 {
-	VERIFY(cond_signal(cv) == 0);
+	ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
+	VERIFY3S(pthread_cond_signal(&cv->cv), ==, 0);
 }
 
 void
 cv_broadcast(kcondvar_t *cv)
 {
-	VERIFY(cond_broadcast(cv) == 0);
+	ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
+	VERIFY3S(pthread_cond_broadcast(&cv->cv), ==, 0);
 }
 
 /*
@@ -571,7 +763,7 @@ __dprintf(const char *file, const char *func, int line, const char *fmt, ...)
 		if (dprintf_find_string("pid"))
 			(void) printf("%d ", getpid());
 		if (dprintf_find_string("tid"))
-			(void) printf("%u ", thr_self());
+			(void) printf("%u ", (uint_t) pthread_self());
 		if (dprintf_find_string("cpu"))
 			(void) printf("%u ", getcpuid());
 		if (dprintf_find_string("time"))
@@ -824,6 +1016,7 @@ kernel_init(int mode)
 	VERIFY((random_fd = open("/dev/random", O_RDONLY)) != -1);
 	VERIFY((urandom_fd = open("/dev/urandom", O_RDONLY)) != -1);
 
+	thread_init();
 	system_taskq_init();
 
 	spa_init(mode);
@@ -835,6 +1028,7 @@ kernel_fini(void)
 	spa_fini();
 
 	system_taskq_fini();
+	thread_fini();
 
 	close(random_fd);
 	close(urandom_fd);
diff --git a/lib/libzpool/taskq.c b/lib/libzpool/taskq.c
index 8db5d11c1..36c0ec7df 100644
--- a/lib/libzpool/taskq.c
+++ b/lib/libzpool/taskq.c
@@ -42,7 +42,7 @@ struct taskq {
 	krwlock_t	tq_threadlock;
 	kcondvar_t	tq_dispatch_cv;
 	kcondvar_t	tq_wait_cv;
-	thread_t	*tq_threadlist;
+	kthread_t	**tq_threadlist;
 	int		tq_flags;
 	int		tq_active;
 	int		tq_nthreads;
@@ -154,7 +154,7 @@ taskq_wait(taskq_t *tq)
 	mutex_exit(&tq->tq_lock);
 }
 
-static void *
+static void
 taskq_thread(void *arg)
 {
 	taskq_t *tq = arg;
@@ -183,7 +183,7 @@ taskq_thread(void *arg)
 	tq->tq_nthreads--;
 	cv_broadcast(&tq->tq_wait_cv);
 	mutex_exit(&tq->tq_lock);
-	return (NULL);
+	thread_exit();
 }
 
 /*ARGSUSED*/
@@ -219,7 +219,7 @@ taskq_create(const char *name, int nthreads, pri_t pri,
 	tq->tq_maxalloc = maxalloc;
 	tq->tq_task.task_next = &tq->tq_task;
 	tq->tq_task.task_prev = &tq->tq_task;
-	tq->tq_threadlist = kmem_alloc(nthreads * sizeof (thread_t), KM_SLEEP);
+	tq->tq_threadlist = kmem_alloc(nthreads*sizeof(kthread_t *), KM_SLEEP);
 
 	if (flags & TASKQ_PREPOPULATE) {
 		mutex_enter(&tq->tq_lock);
@@ -229,8 +229,8 @@ taskq_create(const char *name, int nthreads, pri_t pri,
 	}
 
 	for (t = 0; t < nthreads; t++)
-		(void) thr_create(0, 0, taskq_thread,
-		    tq, THR_BOUND, &tq->tq_threadlist[t]);
+		VERIFY((tq->tq_threadlist[t] = thread_create(NULL, 0,
+		    taskq_thread, tq, TS_RUN, NULL, 0, 0)) != NULL);
 
 	return (tq);
 }
@@ -238,7 +238,6 @@ taskq_create(const char *name, int nthreads, pri_t pri,
 void
 taskq_destroy(taskq_t *tq)
 {
-	int t;
 	int nthreads = tq->tq_nthreads;
 
 	taskq_wait(tq);
@@ -259,10 +258,7 @@ taskq_destroy(taskq_t *tq)
 
 	mutex_exit(&tq->tq_lock);
 
-	for (t = 0; t < nthreads; t++)
-		(void) thr_join(tq->tq_threadlist[t], NULL, NULL);
-
-	kmem_free(tq->tq_threadlist, nthreads * sizeof (thread_t));
+	kmem_free(tq->tq_threadlist, nthreads * sizeof (kthread_t *));
 
 	rw_destroy(&tq->tq_threadlock);
 	mutex_destroy(&tq->tq_lock);
@@ -274,7 +270,7 @@ taskq_destroy(taskq_t *tq)
 }
 
 int
-taskq_member(taskq_t *tq, void *t)
+taskq_member(taskq_t *tq, kthread_t *t)
 {
 	int i;
 
@@ -282,7 +278,7 @@ taskq_member(taskq_t *tq, void *t)
 		return (1);
 
 	for (i = 0; i < tq->tq_nthreads; i++)
-		if (tq->tq_threadlist[i] == (thread_t)(uintptr_t)t)
+		if (tq->tq_threadlist[i] == t)
 			return (1);
 
 	return (0);